rhoconnect 3.0.6 → 3.1.0.beta1

data/bench/lib/bench/distr_runner.rb

@@ -0,0 +1,102 @@
+module Bench
+  class DistrRunner
+  
+    attr_accessor :start_time, :end_time, :clients_group
+    attr_accessor :total_count, :times_count, :times_sum
+    
+    def initialize(clientgroup)
+      @clients_group = clientgroup
+    end
+    
+    def run(server, sync_key, payload, concurrency, niterations, result_filename=nil)
+      # 1) Extract server name
+      server ||= 'default'
+      if server != 'default'
+        Bench.base_url = server
+      end
+      
+      # 2) Simulate the payload
+      Bench.datasize = payload.to_i
+      expected_data = Bench.get_test_data(Bench.datasize)
+      Bench.concurrency = concurrency.to_i
+      Bench.iterations = niterations.to_i
+
+      # 3) extract result filename
+      Bench.result_filename = result_filename
+
+      # 4) Set up the server
+      begin 
+        Bench.admin_login = 'rhoadmin'
+        Bench.admin_password = ''
+        Bench.get_test_server
+        Bench.reset_app
+        Bench.user_name = "benchuser"
+        Bench.password = "password"
+        Bench.set_server_state("test_db_storage:application:#{Bench.user_name}",expected_data)
+        Bench.reset_refresh_time('RhoInternalBenchmarkAdapter')
+
+        #6) set the sync key
+        @start_time = Time.now + 10.0
+        Bench.set_server_state(sync_key, @start_time.to_f.to_s)
+        puts "Sync Key #{sync_key} is set to #{@start_time} - waiting for clients now!!!"
+
+        #7) Start-up all clients
+        command="cd /opt/rhoconnect/bin; ruby run_client_benchmark #{server} #{sync_key} #{niterations} 1 #{Bench.datasize} 1>/dev/null"
+        ec2_clients = clients_group.client_instances[0,concurrency]
+        Bench::AWSUtils.run_stack_ssh_command(ec2_clients, command)
+      
+        #8) Wait until the command is done and save all data to the disk
+        sleep(1)
+        @total_count = 0
+        @start_time = 0.0
+        @end_time = 0.0
+        @times_sum = 0.0
+        @times_count = 0
+        process_results
+        save_results
+
+        #9) Clean-up the db
+        Bench.reset_app
+      rescue Exception => e
+        puts "Distributed Bench Runner ERROR !!!"
+        puts e.message
+        puts e.backtrace.join("\n")
+      end
+    end
+    
+    def process_results
+      finished_data = Bench.get_server_state("bench_statistics_data")
+      finished_data.each do |session,session_data|
+        puts "Client data for #{session} : startTime (#{session_data['start_time']}), endTime (#{session_data['end_time']})"
+        @total_count += session_data['count'].to_i
+        @start_time = session_data['start_time'].to_f unless @start_time > 0.0 and session_data['start_time'].to_f >= @start_time
+        @end_time = session_data['end_time'].to_f unless session_data['end_time'].to_f <= @end_time
+        if not session_data['times'].nil?
+          times_data = session_data['times'].split(',')
+          @times_count += times_data.size
+          times_data.each do |timevalue|
+            @times_sum += timevalue.to_f
+          end
+        end
+      end
+    end
+
+    def average_time
+      @times_count > 0 ? @times_sum/@times_count : 0.0
+    end
+
+    def save_results
+      return if Bench.result_filename.nil?
+
+      res_hash = YAML.load_file(Bench.result_filename) if File.exist?(Bench.result_filename)
+      res_hash ||= {}
+      puts "Overall results are : startTime (#{@start_time.to_s}), endTime (#{@end_time.to_s}), count (#{@total_count.to_s}), payload (#{Bench.datasize}), concurrency (#{Bench.concurrency})"
+      res_hash[Bench.concurrency.to_s] = (@end_time > 0.0 and @start_time > 0.0) ? [@total_count/(@end_time - @start_time)] : [0.0]
+      res_hash[Bench.concurrency.to_s] << average_time
+      File.open(Bench.result_filename, 'w' ) do |file|
+        file.write res_hash.to_yaml unless res_hash.empty?
+      end
+    end
+  
+  end
+end

data/bench/lib/bench/utils.rb

@@ -53,5 +53,132 @@ module Bench
         0
       end
     end
+    
+    def create_subdir(dir)
+      begin
+        if File.directory? dir
+          dir += '_' + Time.now.strftime("%Y_%m_%d_%H_%M_%S")
+        end
+        Dir.mkdir(dir)
+        dir
+      rescue
+      end
+    end
+    
+    def load_settings(file)
+      begin
+        return YAML.load_file(file)
+      rescue Exception => e
+        puts "Error opening settings file #{file}: #{e}."
+        puts e.backtrace.join("\n")
+        raise e
+      end
+    end
+    
+    def start_rhoconnect_app(options)
+      options[:app_started] = false
+      path = options[:start_server] || '.'
+      if not File.exists?(File.join(path,'settings','settings.yml'))
+        puts " Invalid Rhoconnect app path '#{path}' - no app files can be found there"
+        return false
+      end
+      settings = load_settings(File.join(path,'settings','settings.yml'))
+      app_url = settings[:production][:syncserver].gsub(/\/$/,'')
+      
+      uri = URI.parse(app_url)
+      port = (uri.port and uri.port != 80) ? ":"+uri.port.to_s : "" 
+      options[:server_url] = "#{uri.scheme}://#{uri.host}#{port}"
+      
+      already_running = check_rc_service(options[:server_url])
+      if already_running
+        puts " Another Rhoconnect Application is already running at '#{options[:server_url]}'"
+        return false
+      end
+      
+      # start the app now
+      puts "Starting up the Rhoconnect Application at #{options[:server_url]}"
+      current_dir = Dir.pwd
+      Dir.chdir(path)
+      exit_code = system("rake rhoconnect:startbg")
+      sleep 1
+      Dir.chdir(current_dir)
+      
+      options[:app_started] = check_rc_service(options[:server_url])
+    end
+    
+    def stop_rhoconnect_app(options)
+      return unless options[:app_started] == true
+      
+      path = options[:start_server] || '.'
+      if not File.exists?(File.join(path,'settings','settings.yml'))
+        puts " Invalid Rhoconnect app path '#{path}' - no app files can be found there"
+        return
+      end
+      puts "Stopping the Rhoconnect Application at #{options[:server_url]}"
+      current_dir = Dir.pwd
+      Dir.chdir(path)
+      system("rake rhoconnect:stop")
+      sleep 1
+      Dir.chdir(current_dir)
+    end
+    
+    # check_rc_service
+    # Makes an HTTP request to check that the rhoconnect app is up and running
+    def check_rc_service(server_name)
+      success = false
+      begin
+        request = Net::HTTP.get_response(URI.parse("#{server_name}/console"))
+        if request.code == '200'
+          success = true
+        end
+      rescue
+      end
+      success
+    end #check_rc_service
+    
+    # prepare bench results
+    def prepare_bench_results_dirs(dir)
+      start_dir = Dir.pwd
+      Dir.chdir dir
+      
+      # create result directory structure
+      ['bench_results'].each do |dir|
+        next if dir.nil?
+        created_dir = Bench.create_subdir dir
+        Dir.chdir created_dir
+      end
+      result_dir = Dir.pwd
+      ['raw_data'].each do |dir|
+        Bench.create_subdir dir
+      end
+      
+      Dir.chdir start_dir
+      result_dir
+    end
+    
+    def prepare_bench_results_meta(dir, title, x_keys)
+      title ||= "Results"
+      
+      start_dir = Dir.pwd
+      Dir.chdir(dir)
+      
+      # 3) create meta.yml file
+      meta_hash = {:x_keys => {}, :metrics => {}, :label => title}
+      counter = 0
+      x_keys = x_keys.sort_by(&Bench.sort_natural_order)
+      x_keys.each do |x_key|
+        meta_hash[:x_keys][x_key] = counter
+        counter += 1
+      end
+
+      meta_hash[:metrics] = {'Throughput' => 0, 'Av.Time' => 1}
+
+      File.open(File.join(dir,'raw_data','meta.yml'), 'w') do |file|
+        file.write meta_hash.to_yaml unless meta_hash.empty?
+      end
+      
+      Dir.chdir start_dir
+      true
+    end
   end
 end

data/bench/lib/bench.rb

@@ -4,6 +4,7 @@ $:.unshift File.dirname(__FILE__)
 require 'bench/timer'
 require 'bench/logging'
 require 'bench/utils'
+require 'bench/bench_result_processor'
 require 'bench/result'
 require 'bench/session'
 require 'bench/runner'
@@ -53,6 +54,14 @@ module Bench
       end
     end
     
+    def get_server_license_info
+      token = get_token
+      params = {:api_token => token}
+      @body = RestClient.post("#{@host}/api/admin/get_license_info",
+        params.to_json, :content_type => :json)
+      JSON.parse(@body.to_s)
+    end
+    
     def get_server_state(doc)
       token = get_token
       params = {:api_token => token, :doc => doc}
@@ -110,15 +119,18 @@ module Bench
     def get_test_server(app_name = nil)
       if @base_url.nil?
         app_name = 'benchapp' unless app_name
-        load_settings(File.join(File.dirname(__FILE__),'..',app_name,'settings','settings.yml'))
-        @base_url = $settings[:development][:syncserver].gsub(/\/$/,'')
+        settings = load_settings(File.join(File.dirname(__FILE__),'..',app_name,'settings','settings.yml'))
+        @base_url = settings[:development][:syncserver].gsub(/\/$/,'')
       end
       uri = URI.parse(@base_url)
-      port = (uri.port and uri.port != 80) ? ":"+uri.port.to_s : "" 
+      port = (uri.port and uri.port != 80) ? ":"+uri.port.to_s : ""
+      if not uri.scheme
+        raise "URI '#{@base_url}' is invalid:\n\t- doesn't have the required scheme part (for example , http://)"
+      end
       @host = "#{uri.scheme}://#{uri.host}#{port}"
       puts "Test server is #{@host}..."
     end
-  
+    
     def test(&block)
       Runner.new.test(@concurrency,@iterations,&block)
     end
@@ -140,16 +152,5 @@ module Bench
       end
       archive
     end
-    
-    # TODO: Share this code with the framework Rho class
-    def load_settings(file)
-      begin
-        $settings = YAML.load_file(file)
-      rescue Exception => e
-        puts "Error opening settings file #{file}: #{e}."
-        puts e.backtrace.join("\n")
-        raise e
-      end
-    end
   end
 end

data/bench/prepare_bench

@@ -4,27 +4,19 @@ require 'rubygems'
 $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
 require 'bench'
 
-def create_subdir(dir)
-  begin
-    Dir.mkdir(dir)
-  rescue
-  end
-end
-
 # 1) Extract Bench title
 title = ARGV[0]
 
-
 # 2) create result directory structure
 ['bench_results', ARGV[1]].each do |dir|
   next if dir.nil?
-  create_subdir dir
+  Bench.create_subdir dir
   Dir.chdir dir
 end
 result_dir = Dir.pwd
 
-['raw_data', 'images'].each do |dir|
-  create_subdir dir
+['raw_data'].each do |dir|
+  Bench.create_subdir dir
 end
 
 # 3) create meta.yml file

data/bench/scripts/test_query_script.rb

@@ -2,15 +2,14 @@ include BenchHelpers
 bench_log "Runs simple login,clientcreate,clientregister,sync session and validates response"
 
 Bench.config do |config|
-  config.concurrency ||= 10
+  config.concurrency ||= 1
   config.iterations  ||= 10
   config.datasize ||= 100
   config.main_marker = 'get-cud'
   config.user_name = "benchuser"
   config.password = "password"
   config.get_test_server
-  @datasize = Bench.datasize
-  @expected = Bench.get_test_data(@datasize)
+  @datasize = Bench.datasize 
   @all_objects = "[{\"version\":3},{\"token\":\"%s\"},{\"count\":%i},{\"progress_count\":0},{\"total_count\":%i},{\"insert\":""}]"
   @ack_token = "[{\"version\":3},{\"token\":\"\"},{\"count\":0},{\"progress_count\":%i},{\"total_count\":%i},{}]"
   @api_token = Bench.get_token
@@ -19,8 +18,9 @@ Bench.config do |config|
   # if this is not a distributed run - reset the app
   if not Bench.sync_key
     config.reset_app
+    @expected = Bench.get_test_data(@datasize)
     config.set_server_state("test_db_storage:application:#{config.user_name}",@expected)
-    config.reset_refresh_time('MockAdapter')
+    config.reset_refresh_time('RhoInternalBenchmarkAdapter')
   end
 end
 
@@ -34,7 +34,6 @@ Bench.synchronize do |config|
     end
     sleep(0.010)
   end
-  puts " we have here #{cur_time}, #{sync_time}"   
 end
 
 Bench.test do |config,session|
@@ -50,11 +49,11 @@ Bench.test do |config,session|
   end
   
   session.get "get-cud", "#{config.base_url}/query" do
-    {'source_name' => 'MockAdapter', 'client_id' => session.client_id, 'p_size' => @datasize}
+    {'source_name' => 'RhoInternalBenchmarkAdapter', 'client_id' => session.client_id, 'p_size' => @datasize}
   end
   token = JSON.parse(session.last_result.body)[1]['token']
   session.get "ack-cud", "#{config.base_url}/query" do
-    { 'source_name' => 'MockAdapter', 
+    { 'source_name' => 'RhoInternalBenchmarkAdapter', 
       'client_id' => session.client_id,
       'token' => token}
   end

data/bin/rhoconnect-benchmark

@@ -4,10 +4,262 @@ require 'rubygems'
 $:.unshift File.join(File.dirname(__FILE__),'..','bench','lib')
 $:.unshift File.join(File.dirname(__FILE__),'..','bench')
 require 'bench'
+require 'optparse'
 
-$ARGV.unshift ''
-$ARGV.unshift 'rhoadmin'
-$ARGV.unshift 'scripts/test_comm_script.rb'
-$ARGV.unshift 'start'
+ENV['RACK_ENV']='production'
 
-Bench::Cli.start
+RHOCONNECT_PATH = File.join(File.dirname(__FILE__),'..')
+puts "Rhoconnect-Benchmark Script is started at #{Time.now.to_s}"
+
+def parse_options
+  begin 
+    options = {}
+    optparse = OptionParser.new do |opts|
+       # Set a banner, displayed at the top
+       # of the help screen.
+       opts.banner = "Usage: rhoconnect-benchmark [options]"
+
+       options[:server_url] = 'http://localhost:9292'
+       opts.on( '-u', '--url URL', 'Use URL as a server for benchmarking' ) do |url|
+         options[:server_url] = url
+       end
+     
+       options[:start_server] = nil
+       opts.on( '-s', '--startserver [APP_PATH]', 'Starts Rhoconnect App Server on localhost and use it for benchmarking' ) do |path|
+         options[:start_server] = path || '.'
+       end
+     
+       options[:num_clients] = 0
+       opts.on( '-C', '--nclients NUMBER', 'Number of concurrent clients to use' ) do |number|
+         options[:num_clients] = number
+       end
+
+       options[:num_iterations] = 10
+       opts.on( '-R', '--niterations NUMBER', 'Number of iterations to run benchmark test on each client (default - 10)' ) do |number|
+         options[:num_iterations] = number
+       end
+
+       options[:payload] = 100
+       opts.on( '-p', '--payload NUMBER', 'Simulate payload (number of records), default - 100 recs (~20K)' ) do |number|
+         options[:payload] = number
+       end
+       
+       options[:max_payload] = 100
+       opts.on( '-x', '--mapayload NUMBER', 'Run the benchmarks test varying the payload from 1 to NUMBER' ) do |number|
+         options[:max_payload] = number
+         options[:payload] = number
+       end
+       
+       options[:max_clients] = 1
+       opts.on( '-m', '--maxclientloops NUMBER', 'Run the benchmarks test varying the number of clients from 1 to NUMBER' ) do |number|
+         options[:max_clients] = number
+         options[:num_clients] = number
+       end
+
+       options[:max_payload] = 0
+       opts.on( '-x', '--mapayload NUMBER', 'Run the benchmarks test varying the payload from 1 to NUMBER' ) do |number|
+         options[:max_payload] = number
+         options[:payload] = number
+       end
+       
+       options[:save_results] = nil
+       options[:title] = 'QUERY'
+       opts.on( '-S', '--save [PATH]', 'Save all benchmark results in to the file structure at PATH') do |path|
+         options[:save_results] = path || '.'
+         options[:title] ||= 'QUERY'
+       end
+
+       opts.on('-t', '--title TITLE', "Name this benchmark's test results with the title TITLE") do |title|
+         options[:title] = title
+       end
+       
+       opts.on( '-A', '--advanced', 'This convinience option combines -m 10 and -x 500') do
+         options[:max_payload] = 500
+         options[:payload] = 500
+         options[:max_clients] = 10
+         options[:num_clients] = 10
+         options[:title] ||= "QUERY"
+         options[:save_results] = '.'
+       end
+       
+       options[:distributed] = nil
+       opts.on('-D', '--distributed FNAME', 'Run benchmark test from distributed AWS Cloud Formation using FNAME for options') do |fname|
+         options[:max_payload] = 500
+         options[:payload] = 500
+         options[:max_clients] = 10
+         options[:num_clients] = 10
+         options[:title] ||= "QUERY"
+         options[:save_results] = '.'
+         options[:distributed] = fname
+       end
+          
+       # This displays the help screen, all programs are
+       # assumed to have this option.
+       opts.on( '-h', '--help', 'Display rhoconnect-benchmark help information' ) do
+         puts opts
+         exit 0
+       end
+     end
+     
+     optparse.parse!
+   rescue SystemExit => se
+     exit 0
+   rescue Exception => e
+     puts " Error while parsing the command line options: #{e.message}"
+     puts ""
+     puts optparse.help
+     puts ""
+     raise ""
+   end
+   
+   options
+end
+
+begin
+  options = parse_options
+  # 1) start up the application - if required
+  # NOTE: server URL will be updated
+  # from the app's settings
+  if options[:start_server] != nil
+    if not Bench.start_rhoconnect_app(options)
+      puts " Cannot start rhoconnect application at path '#{options[:start_server]}'"
+      puts " Exiting..."
+      exit 10
+    end
+  end
+  
+  # 2) Now, parse the server's options
+  Bench.base_url = "#{options[:server_url]}/api/application"
+  Bench.get_test_server
+  Bench.admin_login = 'rhoadmin'
+  Bench.admin_password = ''
+  #
+  if not Bench.check_rc_service(Bench.host)
+    puts ""
+    puts " In order to run benchmark tests - you need to have the Rhoconnect application"
+    puts " up and running on #{options[:server_url]}"
+    puts ""
+    exit 10
+  end
+  
+  # get license info from the server
+  rc_license_info = Bench.get_server_license_info
+    
+  if options[:num_clients] == 0
+    n_clients = rc_license_info['seats'].to_i / options[:num_iterations]
+    if n_clients > 20
+      n_clients = 20
+    end
+    options[:num_clients] = n_clients
+  end
+  # calculate appropriate numbers for concurrency  
+  total_num_clients = options[:num_iterations].to_i*options[:num_clients].to_i
+  if total_num_clients > rc_license_info['seats']
+    puts ""
+    puts " In order to run benchmark tests with #{options[:num_clients]} concurrent clients"
+    puts " for #{options[:num_iterations]} iterations you need to have a license with at least #{total_num_clients} seats."
+    puts " Your application have only #{rc_license_info['seats']} seats available."
+    puts " Adjust the number of concurrent clients or number of iterations to fit within the available license."
+    puts ""
+    exit 10
+  end
+  
+  # run test in a loop from 1 to max_clients concurrency
+  concurrency_array = []
+  if options[:max_clients].to_i > 1
+    concurrency_array << 1
+    num_loops = 5
+    loop_step = (options[:max_clients].to_f - 1.to_f)/(num_loops - 1)
+    for i in 1..num_loops-2
+      concurrency_array << (1 + i*loop_step + 0.5).to_i 
+    end 
+  end
+  concurrency_array << options[:num_clients].to_i
+  concurrency_array.uniq!
+  
+  # run test in a loop from 1 to max_payload records
+  payload_array = []
+  if options[:max_payload].to_i > 1
+    payload_array << 1
+    num_loops = 10
+    loop_step = (options[:max_payload].to_f - 1.to_f)/(num_loops - 1)
+    for i in 1..num_loops-2
+      payload_array << (1 + i*loop_step + 0.5).to_i 
+    end 
+  end
+  payload_array << options[:payload].to_i
+  payload_array.uniq!
+  
+  # load distributed options and create the AWS Cloud Formation Stack
+  if options[:distributed] != nil
+    puts ""
+    puts " Creating Distributed Clients Benchmark"
+    puts ""
+    require 'bench/aws_utils'
+    require 'bench/distr_runner'
+    Bench::AWSUtils.init_connection(options[:distributed])
+    options[:clients_group] = nil
+    options[:clients_group] = Bench::AWSUtils.create_cf_stack
+    exit 10 unless options[:clients_group] 
+    options[:distr_runner] = Bench::DistrRunner.new options[:clients_group]
+  end
+  
+  # load the save results options
+  result_dir = nil
+  if options[:save_results]
+    result_dir = Bench.prepare_bench_results_dirs(options[:save_results])
+    x_keys = concurrency_array.collect {|t| t.to_s }
+    Bench.prepare_bench_results_meta(result_dir, options[:title], x_keys)
+  end
+   
+  concurrency_array.each do |concurrency|
+    payload_array.each do |payload|
+      # initialize default parameters 
+      bench_args = []
+      # Thor's task
+      bench_args[0] = 'start'
+      # script
+      bench_args[1] = 'scripts/test_query_script.rb'
+      # admin
+      bench_args[2] = 'rhoadmin'
+      # admin's password
+      bench_args[3] = ''
+      # server's URL
+      bench_args[4] = Bench.base_url
+      # result fname
+      result_fname = nil
+      result_fname = File.join(result_dir,'raw_data',"bench_result.#{payload}") if result_dir
+      bench_args[5] = result_fname
+      # concurrency
+      bench_args[6] = concurrency
+      # number of iterations
+      bench_args[7] = options[:num_iterations]
+      # datasize
+      bench_args[8] = payload
+  
+      # now execute Bench
+      if(options[:distributed])
+        sync_key = "#{options[:title]}_#{payload}"
+        options[:distr_runner].run(Bench.base_url, sync_key, payload, concurrency, options[:num_iterations], result_fname)
+      else
+        Bench::Cli.start bench_args
+      end
+    end
+  end
+  
+  # once finished - post-process results
+  Bench::PostProcessing.execute result_dir
+rescue Exception => e
+  puts e.message
+ensure
+  if options and options[:app_started] == true
+    Bench.stop_rhoconnect_app(options)
+  end
+  
+  if options and options[:clients_group]
+    Bench::AWSUtils.delete_cf_stack(options[:clients_group].stack_name)
+  end 
+end
+
+puts "Rhoconnect-Benchmark Script is finished at #{Time.now.to_s}"
+puts ""