rhoconnect 3.0.6 → 3.1.0.beta1

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.1.0.beta1 (2011-12-31)
+* #20396499 - saving rhoconnect-benchmark results into the file
+* #21630639 - rhoconnect-benchmark post-processing (GRUFF png images and EXCEL spreadsheets generation)
+* #21363347 - rhoconnect-benchmark - support for varying the number of concurrent clients and payload, distributed AWS clients
+* #20151369 - rhoconnect-benchmark command
+* #21187827 - support for built-in internal adapters
+* #22208995 - AWS Cloud Formation deployment guidelines docs
+* #22209097 - Conflict Resolution framework for simultaneous CUD operations
+
 ## 3.0.6 (2011-11-15)
 * #20022889 - support for GET/POST /api/application routes
 * #20609683 - support 'append' parameter in 'set_db_doc' REST API

data/Gemfile

@@ -11,15 +11,15 @@ platforms :jruby do
   gem 'warbler'
 end
 
-gem 'sqlite3', ">= 1.3.3", :platforms => [:ruby, :mswin, :mingw]
-
 group :development do
   gem 'aws-s3', '>= 0.6.2', :require => 'aws/s3'
+  gem 'sqlite3', ">= 1.3.3", :platforms => [:ruby, :mswin, :mingw]
   gem 'fpm', '>= 0.3.8'
-  gem 'fog', '>= 1.0.0' 
+  gem 'fog' 
   gem 'ffaker', '~> 1.8.0'
   gem 'thor', '>= 0.13.6'
   gem 'webmock', '~> 1.6.4'
+  gem 'SystemTimer', '~> 1.2.3', :platforms => :ruby_18
 end
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rhoconnect (3.0.6)
+    rhoconnect (3.1.0.beta1)
       bundler (~> 1.0)
       json (~> 1.5.0)
       rake (~> 0.9.2)
@@ -16,45 +16,59 @@ PATH
 GEM
   remote: http://rubygems.org/
   specs:
+    SystemTimer (1.2.3)
     addressable (2.2.6)
     aws-s3 (0.6.2)
       builder
       mime-types
       xml-simple
+    bouncy-castle-java (1.5.0146.1)
     builder (3.0.0)
     crack (0.3.1)
+    dbd-jdbc (0.1.4)
+    dbi (0.4.5)
+      deprecated (= 2.0.1)
+    deprecated (2.0.1)
     diff-lcs (1.1.3)
-    excon (0.7.6)
+    excon (0.7.12)
     extlib (0.9.15)
     ffaker (1.8.1)
-    fog (1.0.0)
+    fog (1.1.1)
       builder
-      excon (~> 0.7.3)
+      excon (~> 0.7.4)
       formatador (~> 0.2.0)
       mime-types
       multi_json (~> 1.0.3)
       net-scp (~> 1.0.4)
-      net-ssh (~> 2.1.4)
+      net-ssh (>= 2.1.3)
       nokogiri (~> 1.5.0)
       ruby-hmac
     formatador (0.2.1)
-    fpm (0.3.10)
+    fpm (0.3.11)
       json
-    highline (1.6.2)
+    highline (1.6.8)
+    jdbc-sqlite3 (3.7.2)
+    jruby-jars (1.6.5)
+    jruby-openssl (0.7.4)
+      bouncy-castle-java
+    jruby-rack (1.1.1)
     json (1.5.4)
-    mime-types (1.17.1)
-    multi_json (1.0.3)
+    json (1.5.4-java)
+    mime-types (1.17.2)
+    multi_json (1.0.4)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
-    net-ssh (2.1.4)
+    net-ssh (2.2.1)
     nokogiri (1.5.0)
+    nokogiri (1.5.0-java)
     rack (1.3.5)
     rack-protection (1.1.4)
       rack
     rack-test (0.6.1)
       rack (>= 1.0)
-    rake (0.9.2)
+    rake (0.9.2.2)
     rcov (0.9.11)
+    rcov (0.9.11-java)
     redis (2.2.2)
     redis-namespace (1.0.3)
       redis (< 3.0.0)
@@ -74,12 +88,12 @@ GEM
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.6.0)
     ruby-hmac (0.4.0)
-    rubyzip (0.9.4)
+    rubyzip (0.9.5)
     sinatra (1.3.1)
-      rack (>= 1.3.4, ~> 1.3)
-      rack-protection (>= 1.1.2, ~> 1.1)
-      tilt (>= 1.3.3, ~> 1.3)
-    sqlite3 (1.3.4)
+      rack (~> 1.3, >= 1.3.4)
+      rack-protection (~> 1.1, >= 1.1.2)
+      tilt (~> 1.3, >= 1.3.3)
+    sqlite3 (1.3.5)
     templater (1.0.0)
       diff-lcs (>= 1.1.2)
       extlib (>= 0.9.5)
@@ -89,20 +103,27 @@ GEM
     uuidtools (2.1.2)
     vegas (0.1.8)
       rack (>= 1.0.0)
+    warbler (1.3.2)
+      jruby-jars (>= 1.4.0)
+      jruby-rack (>= 1.0.0)
+      rake (>= 0.8.7)
+      rubyzip (>= 0.9.4)
     webmock (1.6.4)
       addressable (~> 2.2, > 2.2.5)
       crack (>= 0.1.7)
     xml-simple (1.1.1)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
+  SystemTimer (~> 1.2.3)
   aws-s3 (>= 0.6.2)
   dbd-jdbc (>= 0.1.4)
   dbi (>= 0.4.5)
   ffaker (~> 1.8.0)
-  fog (>= 1.0.0)
+  fog
   fpm (>= 0.3.8)
   jdbc-sqlite3 (>= 3.7.2)
   jruby-openssl (>= 0.7.4)

data/Rakefile

@@ -155,11 +155,6 @@ task "build:deb" => :build do
   build_pkg "deb", "all", DEB_DEPS
 end
 
-desc "Build Debian DEB rhoconnect-#{Rhoconnect::VERSION}_all.deb package into the pkg directory"
-task "build:bench:deb" => :build do
-  build_pkg "deb", "all", DEB_DEPS, true
-end
-
 RPM_DEPS = [
   "wget >= 1.0", "make >= 3.0", "patch >= 2.0",
   "gcc-c++ >= 4.1.2",
@@ -174,8 +169,3 @@ desc "Build Red Hat RPM rhoconnect-#{Rhoconnect::VERSION}.noarch.rpm package int
 task "build:rpm" => :build do
   build_pkg "rpm", "noarch", RPM_DEPS
 end
-
-desc "Build Red Hat RPM benchmark rhoconnect-#{Rhoconnect::VERSION}.noarch.rpm package into the pkg directory"
-task "build:bench:rpm" => :build do
-  build_pkg "rpm", "noarch", RPM_DEPS, true
-end

data/bench/benchapp/Gemfile.lock

@@ -18,6 +18,7 @@ GEM
       bouncy-castle-java
     jruby-rack (1.0.10)
     json (1.5.4)
+    json (1.5.4-java)
     mime-types (1.17.2)
     multi_json (1.0.3)
     rack (1.3.5)

data/bench/distr_bench/distr_bench_main

@@ -2,7 +2,9 @@
 
 require 'rubygems'
 $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib'))
+require 'net/ssh/multi'
 require 'bench'
+require 'bench/aws_utils'
 
 def process_results
   finished_data = Bench.get_server_state("bench_statistics_data")
@@ -56,33 +58,97 @@ if server != 'default'
   Bench.base_url = server
 end
 
-# 5) Set up the server
-Bench.admin_login = 'rhoadmin'
-Bench.admin_password = ''
-Bench.get_test_server
-Bench.reset_app
-Bench.user_name = "benchuser"
-Bench.password = "password"
-Bench.set_server_state("test_db_storage:application:#{Bench.user_name}",@expected)
-Bench.reset_refresh_time('MockAdapter')
-
-#6) set the sync key
-@start_time = Time.now + 5.0
-Bench.set_server_state(sync_key, @start_time.to_f.to_s)
-puts "Sync Key #{sync_key} is set to #{@start_time} - Please, start the clients now!!!"
-
-#7) wait until all distributed clients are finished
-while true
-  sleep(1)
-  stats_data = Bench.get_server_state("bench_statistics_data")
-  if (not stats_data.nil?) and stats_data.size == Bench.concurrency
-    break
-  end
-  puts " Waiting for the clients to finish : #{Bench.concurrency - stats_data.size} out of #{Bench.concurrency}"
+# 5) Create server EC2 instance
+@fog = make_fog
+server_stack = Constants::CENTOS_STACK
+server_stack[:flavor_id] = 'm1.large'
+
+puts "\n" +
+"================================================\n" +
+"          Creating Benchmark Server             \n" +
+"================================================\n" +
+"\n\n"
+
+@server = create_ec2_instance(@fog, server_stack)
+if @server.nil?
+  puts " Can not create Benchmark Server - Exiting..."
+  exit 7
+end
+
+@server_dist = compile_stack_info('centos')
+
+# Establish connections
+@server_ssh = establish_ssh_connection(@server.dns_name, @server_dist[:remote_user])
+@server_dist[:remote_home] = @server_ssh.run("echo ~")[0].stdout.strip
+
+install_package(@server_dist, @server_ssh)
+
+start_servers(@server_ssh)
+
+success = check_rc_service(@server.dns_name)
+
+puts " success is #{success}"
+
+if not success 
+  exit 10
 end
 
-# 8) Now, save all data to the disk
+destroy_ec2_instance(@server)
+exit 1
+
+# 5) Set up the server
 begin 
+  Bench.admin_login = 'rhoadmin'
+  Bench.admin_password = ''
+  Bench.get_test_server
+  Bench.reset_app
+  Bench.user_name = "benchuser"
+  Bench.password = "password"
+  Bench.set_server_state("test_db_storage:application:#{Bench.user_name}",@expected)
+  Bench.reset_refresh_time('MockAdapter')
+
+  #6) set the sync key
+  @start_time = Time.now + 5.0
+  server_key = "#{sync_key}_#{Bench.datasize}"
+  Bench.set_server_state(server_key, @start_time.to_f.to_s)
+  puts "Sync Key #{server_key} is set to #{@start_time} - waiting for clients now!!!"
+
+  #7) Start-up all clients
+  n_iterations = 50
+  command="cd /opt/rhoconnect/bench/distr_bench; ./run_distr_client.sh #{sync_key} #{server} #{n_iterations} #{Bench.datasize} 1>/dev/null"
+
+  clients = ['ec2-107-20-56-25.compute-1.amazonaws.com',
+    'ec2-107-20-56-25.compute-1.amazonaws.com',
+    'ec2-67-202-35-54.compute-1.amazonaws.com',
+    'ec2-50-17-104-6.compute-1.amazonaws.com',
+    'ec2-204-236-222-92.compute-1.amazonaws.com',
+    'ec2-50-17-92-146.compute-1.amazonaws.com',
+    'ec2-107-22-124-158.compute-1.amazonaws.com',
+    'ec2-184-72-165-204.compute-1.amazonaws.com',
+    'ec2-107-20-41-32.compute-1.amazonaws.com',
+    'ec2-107-22-35-238.compute-1.amazonaws.com',
+    'ec2-50-16-180-250.compute-1.amazonaws.com']
+
+  sess_options = {:keys => ['/tmp/EC2-inst.pem']}
+  Net::SSH::Multi.start({:default_user => 'ec2-user'}) do |session|
+    # define the servers we want to use
+    session.use(sess_options) { clients[0..Bench.concurrency] }
+  
+    # execute commands on all servers
+    session.exec command
+  end
+  
+  #8) wait until all distributed clients are finished
+  while true
+    sleep(1)
+    stats_data = Bench.get_server_state("bench_statistics_data")
+    if (not stats_data.nil?) and stats_data.size == Bench.concurrency
+      break
+    end
+    puts " Waiting for the clients to finish : #{Bench.concurrency - stats_data.size} out of #{Bench.concurrency}"
+  end
+
+  # 9) Now, save all data to the disk
   sleep(1)
   @total_count = 0
   @start_time = 0.0
@@ -91,9 +157,10 @@ begin
   @times_count = 0
   process_results
   save_results
+  
+  #10) Clean-up the db
+  Bench.reset_app
 rescue Exception => e
+  puts "Distributed Bench Run-time ERROR!!!"
   puts e
 end
-
-#8) Clean-up the db
-Bench.reset_app

data/bench/distr_bench/run_test_query_script.sh

@@ -11,18 +11,20 @@ if [ $# -gt 1 ] ; then
     server=$2
 fi
 
-clients[0]=ec2-50-17-151-136.compute-1.amazonaws.com
-clients[1]=ec2-107-22-43-25.compute-1.amazonaws.com
-clients[2]=ec2-75-101-174-0.compute-1.amazonaws.com
-clients[3]=ec2-50-16-87-55.compute-1.amazonaws.com
-clients[4]=ec2-107-22-48-222.compute-1.amazonaws.com
-clients[5]=ec2-107-22-61-93.compute-1.amazonaws.com
-clients[6]=ec2-107-22-53-145.compute-1.amazonaws.com
-clients[7]=ec2-50-17-152-123.compute-1.amazonaws.com
-clients[8]=ec2-50-17-155-181.compute-1.amazonaws.com
-clients[9]=ec2-75-101-225-80.compute-1.amazonaws.com
-
-n_iterations=100
+
+
+clients[0]=ec2-107-20-56-25.compute-1.amazonaws.com
+clients[1]=ec2-67-202-35-54.compute-1.amazonaws.com
+clients[2]=ec2-50-17-104-6.compute-1.amazonaws.com
+clients[3]=ec2-204-236-222-92.compute-1.amazonaws.com
+clients[4]=ec2-50-17-92-146.compute-1.amazonaws.com
+clients[5]=ec2-107-22-124-158.compute-1.amazonaws.com
+clients[6]=ec2-184-72-165-204.compute-1.amazonaws.com
+clients[7]=ec2-107-20-41-32.compute-1.amazonaws.com
+clients[8]=ec2-107-22-35-238.compute-1.amazonaws.com
+clients[9]=ec2-50-16-180-250.compute-1.amazonaws.com
+
+n_iterations=25
 current_path=`pwd`
 
 # setup the benchmark directory structure
@@ -35,14 +37,16 @@ do
 	do
 		result_filename=./bench_results/query_bench_$timestamp_postfix/raw_data/query_bench_result.$payload
 		sync_key=sync_${timestamp_postfix}
-		for ((  i = 0 ;  i < $n_clients;  i++  ))
-		do
-			ssh -i /tmp/EC2-inst.pem -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" ec2-user@${clients[$i]} "cd /opt/rhoconnect/bench/distr_bench; ./run_distr_client.sh $sync_key $server $n_iterations $payload 1>/dev/null" &
-		done
+#		for ((  i = 0 ;  i < $n_clients;  i++  ))
+#		do
+#			ssh -i /tmp/EC2-inst.pem -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" ec2-user@${clients[$i]} "cd /opt/rhoconnect/bench/distr_bench; ./run_distr_client.sh $sync_key $server $n_iterations $payload 1>/dev/null" &
+#			started_pid=$!
+#			echo $started_pid
+#		done
 	
 		# start main script
-		ruby distr_bench_main ${sync_key}_$payload $payload $n_clients $result_filename $server
-		sleep 5
+		ruby distr_bench_main $sync_key $payload $n_clients $result_filename $server
+		sleep 10	
 	done
 done