virtualmonkey 0.0.1

data/lib/virtualmonkey/deployment_runner.rb

@@ -0,0 +1,333 @@
+module VirtualMonkey
+  module DeploymentRunner
+    include VirtualMonkey::TestCaseInterface
+    attr_accessor :deployment, :servers
+    attr_accessor :scripts_to_run
+    
+    def initialize(deployment)
+      @deployment = Deployment.find_by_nickname_speed(deployment).first
+      raise "Fatal: Could not find a deployment named #{deployment}" unless @deployment
+      @servers = @deployment.servers_no_reload
+      @scripts_to_run = {}
+      @rerun_last_command = []
+      lookup_scripts
+      @servers.each { |s| s.settings }
+    end
+
+    # It's not that I'm a Java fundamentalist; I merely believe that mortals should
+    # not be calling the following methods directly. Instead, they should use the
+    # TestCaseInterface methods (behavior, verify, probe) to access these functions.
+    # Trust me, I know what's good for you. -- Tim R.
+    private
+
+    def lookup_scripts
+      puts "WARNING: lookup_scripts is undefined, this must be set in mixin classes"
+      scripts = [
+                 [ 'script_ref1', 'name' ],
+                 [ 'script_ref2', 'name' ]
+               ]
+#      st = ServerTemplate.find(s_two.server_template_href)
+#      lookup_scripts_table(st,scripts)
+    end
+
+    def lookup_scripts_table(st,table)
+      table.each { |a|
+        @scripts_to_run[ a[0] ] = st.executables.detect { |ex| ex.name =~ /#{a[1]}/ }
+        raise "FATAL: Script #{a[1]} not found" unless @scripts_to_run[ a[0] ]
+      }
+    end
+
+    def s_one
+      @servers[0]
+    end
+
+    def s_two
+      @servers[1]
+    end
+
+    def s_three
+      @servers[2]
+    end
+
+    def s_four
+      @servers[3]
+    end
+
+    # Launch all servers in the deployment.
+    def launch_all
+      @servers.each { |s|
+        begin
+          object_behavior(s, :start)
+        rescue Exception => e
+          raise e unless e.message =~ /AlreadyLaunchedError/
+        end
+      }
+    end
+
+    # sets the MASTER_DB_DNSNAME to this machine's ip address
+    def set_master_db_dnsname
+      the_name = get_tester_ip_addr
+      @deployment.set_input("MASTER_DB_DNSNAME", the_name) 
+      @deployment.set_input("DB_HOST_NAME", the_name) 
+    end
+
+    # Helper method, performs selection of a subset of servers to operate on based on the server's nicknames.
+    # * nickname_substr<~String> - regex compatible string to match
+    def select_set(nickname_substr)
+      @servers.select { |s| s.nickname =~ /#{nickname_substr}/ }
+    end
+
+    # Launch server(s) that match nickname_substr
+    # * nickname_substr<~String> - regex compatible string to match
+    def launch_set(nickname_substr)
+      set = select_set(nickname_substr)  
+      set.each { |s|
+        begin
+          object_behavior(s, :start)
+        rescue Exception => e
+          raise e unless e.message =~ /AlreadyLaunchedError/
+        end
+      }
+    end
+
+    # Re-Launch all server
+    def relaunch_all
+      @servers.each { |s|
+        begin
+          object_behavior(s, :relaunch)
+        rescue Exception => e
+          raise e #unless e.message =~ /AlreadyLaunchedError/
+        end
+      }
+    end
+
+    # un-set all tags on all servers in the deployment
+    def unset_all_tags
+      @deployment.servers_no_reload.each do |s|
+        # can't unset ALL tags, so we must set a bogus one
+        s.tags = [{"name"=>"removeme:now=1"}]
+        s.save
+      end
+    end
+
+    # Wait for server(s) matching nickname_substr to enter state
+    # * nickname_substr<~String> - regex compatible string to match
+    # * state<~String> - state to wait for, eg. operational
+    def wait_for_set(nickname_substr, state)
+      set = select_set(nickname_substr)  
+      state_wait(set, state)
+    end
+
+    # Wait for server(s) matching nickname_substr to enter state
+    # * servers<~Array> - Array of Servers to wait on
+    # * state<~String> - state to wait for, eg. operational
+    def wait_for_servers(servers, state)
+      state_wait(set, state)
+    end
+
+    # Helper method, waits for state on a set of servers.
+    # * set<~Array> of servers to operate on
+    # * state<~String> state to wait for
+    def state_wait(set, state)
+      # do a special wait, if waiting for operational (for dns)
+      if state == "operational"
+        set.each { |server| server.wait_for_operational_with_dns }
+      else
+        set.each { |server| server.wait_for_state(state) }
+      end
+    end
+    
+    # Wait for all server(s) to enter state.
+    # * state<~String> - state to wait for, eg. operational
+    def wait_for_all(state)
+      state_wait(@servers, state)
+    end
+
+    def start_ebs_all(wait=true)
+      @servers.each { |s| s.start_ebs }
+      wait_for_all("operational") if wait
+      @servers.each { |s| 
+        s.dns_name = nil 
+        s.private_dns_name = nil
+        }
+    end
+
+    def stop_ebs_all(wait=true)
+      @servers.each { |s| s.stop_ebs }
+      wait_for_all("stopped") if wait
+      @servers.each { |s| 
+        s.dns_name = nil 
+        s.private_dns_name = nil
+        }
+    end
+
+    def stop_all(wait=true)
+      @servers.each { |s| s.stop }
+      wait_for_all("stopped") if wait
+      @servers.each { |s| 
+        s.dns_name = nil 
+        s.private_dns_name = nil
+        }
+    end
+
+    def reboot_all(serially_reboot = false)
+      wait_for_reboot = true
+      # Do NOT thread this each statement
+      @servers.each do |s| 
+        object_behavior(s, :reboot, wait_for_reboot)
+        if serially_reboot
+          object_behavior(s, :wait_for_state, "operational")
+        end
+      end
+      @servers.each do |s| 
+        object_behavior(s, :wait_for_state, "operational")
+      end
+    end
+
+    # Run a script on server in the deployment
+    # * server<~Server> the server to run the script on
+    # * friendly_name<~String> string lookup for Hash @scripts_to_run.  @scripts_to_run must be a Hash containing an Executable with this key.
+    def run_script(friendly_name, server)
+      audit = server.run_executable(@scripts_to_run[friendly_name])
+      audit.wait_for_completed
+    end
+
+    
+    # Detect operating system on each server and stuff the corresponding values for platform into the servers params (for temp storage only)
+    def detect_os
+      @server_os = Array.new
+      @servers.each do |server|
+        if object_behavior(server, :spot_check_command?, "lsb_release -is | grep Ubuntu")
+          puts "setting server to ubuntu"
+          server.os = "ubuntu"
+          server.apache_str = "apache2"
+          server.apache_check = "apache2ctl status"
+          server.haproxy_check = "service haproxy status"
+        else
+          puts "setting server to centos"
+          server.os = "centos"
+          server.apache_str = "httpd"
+          server.apache_check = "service httpd status"
+          server.haproxy_check = "service haproxy check"
+        end
+      end
+    end
+    
+    # Assumes the host machine is EC2, uses the meta-data to grab the IP address of this
+    # 'tester server' eg. used for the input variation MASTER_DB_DNSNAME
+    def get_tester_ip_addr
+      if File.exists?("/var/spool/ec2/meta-data.rb")
+        require "/var/spool/ec2/meta-data-cache" 
+      else
+        ENV['EC2_PUBLIC_HOSTNAME'] = "127.0.0.1"
+      end
+      my_ip_input = "text:" 
+      my_ip_input += ENV['EC2_PUBLIC_HOSTNAME']
+      my_ip_input
+    end
+    
+    # Log rotation
+    def force_log_rotation(server)
+      response = server.spot_check_command?('logrotate -f /etc/logrotate.conf')
+      raise "Logrotate restart failed" unless response
+    end
+    
+    def log_check(server, logfile)
+      response = nil
+      count = 0
+      until response || count > 3 do
+        response = server.spot_check_command?("test -f #{logfile}")
+        break if response
+        count += 1
+        sleep 10
+      end
+      raise "Log file does not exist: #{logfile}" unless response
+    end   
+
+    # Checks that monitoring is enabled on all servers in the deployment.  Will raise an error if monitoring is not enabled.
+    def check_monitoring
+      @servers.each do |server|
+        server.settings
+        response = nil
+        count = 0
+        until response || count > 20 do
+          begin
+            response = server.monitoring
+          rescue
+            response = nil
+            count += 1
+            sleep 10
+          end
+        end
+        raise "Fatal: Failed to verify that monitoring is operational" unless response
+#TODO: pass in some list of plugin info to check multiple values.  For now just
+# hardcoding the cpu check
+        sleep 60 # This is to allow monitoring data to accumulate
+        monitor=server.get_sketchy_data({'start'=>-60,'end'=>-20,'plugin_name'=>"cpu-0",'plugin_type'=>"cpu-idle"})
+        idle_values = monitor['data']['value']
+        raise "No cpu idle data" unless idle_values.length > 0
+        raise "No idle time" unless idle_values[0] > 0
+        puts "Monitoring is OK for #{server.nickname}"
+      end
+    end
+
+
+
+# TODO - we do not know what the RS_INSTANCE_ID available to the testing.
+# For now we are checking at a high level that the services are working
+# and then assume that the config file changes done during start are
+# correct for the new instance data.
+#
+    def perform_start_stop_operations
+      behavior(:detect_os)
+      s=@servers.first
+      # Save configuration files for comparison after starting
+      behavior(:save_configuration_files, s)
+      # Stop the servers
+      behavior(:stop_ebs_all)
+      # Verify all stopped
+      # Start the servers
+      behavior(:start_ebs_all, true)
+#      Do this for all? Or just the one?
+#      @servers.each { |server| server.wait_for_operational_with_dns }
+      s=@servers.first
+      object_behavior(s, :wait_for_operational_with_dns)
+      # Verify operational
+      behavior(:run_simple_check, s)
+    end
+
+# TODO there will be other files that need compares et al.  Create a list
+# of them and abstarct the tests
+    # Copy configuration files into some location for usage after start
+    def save_configuration_files(server)
+      puts "Saving config files"
+      object_behavior(server, :spot_check_command, 'mkdir -p /root/start_stop_backup')
+      object_behavior(server, :spot_check_command, 'cp /etc/postfix/main.cf /root/start_stop_backup/.')
+      object_behavior(server, :spot_check_command, 'cp /etc/syslog-ng/syslog-ng.conf /root/start_stop_backup/.')
+    end
+
+    # Diff the new config file with the saved one and check that the only
+    # line that is different is the one that has the mydestination change
+    def test_mail_config(server)
+      res = server.spot_check_command('diff /etc/postfix/main.cf /root/start_stop_backup/main.cf')
+# This is lame - assuming if the file is modified then it's okay
+        raise "ERROR: postfix main.cf configuration file did not change when restarted" unless res
+    end
+    
+    def test_syslog_config(server)
+      res = server.spot_check_command('diff /etc/syslog-ng/syslog-ng.conf /root/start_stop_backup/syslog-ng.conf')
+# This is lame - assuming if the file is modified then it's okay
+      raise "ERROR: syslog-ng configuration file did not change when restarted" unless res
+    end
+    
+    def run_simple_checks
+      @servers.each { |s| behavior(:run_simple_check, s) }
+    end
+    
+    # this is where ALL the generic application server checks live, this could get rather long but for now it's a single method with a sequence of checks
+    def run_simple_check(server)
+      behavior(:test_mail_config, server)
+      behavior(:test_syslog_config, server)
+    end
+  end
+end

data/lib/virtualmonkey/ebs.rb

@@ -0,0 +1,161 @@
+module VirtualMonkey
+  module EBS
+    include VirtualMonkey::DeploymentRunner
+    attr_accessor :stripe_count
+    attr_accessor :volume_size
+    attr_accessor :mount_point
+    attr_accessor :lineage
+
+    # sets the stripe count for the deployment
+    # * count<~String> eg. "3"
+    def set_variation_stripe_count(count)
+      @stripe_count = count
+      @deployment.set_input("EBS_STRIPE_COUNT", "text:#{@stripe_count}")
+    end
+
+    # sets the volume size n GB for the runner
+    # * kind<~Num> 
+    def set_variation_volume_size(size)
+      @volume_size = size
+    end
+
+    # sets the EBS mount point for the runner
+    # * kind<~String> 
+    def set_variation_mount_point(mnt)
+      @mount_point = mnt
+    end
+
+    # sets the lineage for the deployment
+    # * kind<~String> can be "chef" or nil
+    def set_variation_lineage(kind = nil)
+      @lineage = "testlineage#{@deployment.href.split(/\//).last}"
+      if kind
+        raise "Only support nil kind for ebs lineage"
+      else
+        @deployment.set_input('EBS_LINEAGE', "text:#{@lineage}")
+        # unset all server level inputs in the deployment to ensure use of 
+        # the setting from the deployment level
+        @deployment.servers_no_reload.each do |s|
+          s.set_input('EBS_LINEAGE', "text:")
+        end
+      end
+    end
+
+    # take the lineage name, find all snapshots and sleep until none are in the pending state.
+    def wait_for_snapshots
+      timeout=1500
+      step=10
+      while timeout > 0
+        puts "Checking for snapshot completed"
+        snapshots = behavior(:find_snapshots)
+        status= snapshots.map { |x| x.aws_status } 
+        break unless status.include?("pending")
+        sleep step
+        timeout -= step
+      end
+      raise "FATAL: timed out waiting for all snapshots in lineage #{@lineage} to complete" if timeout == 0
+    end
+
+    # Find all snapshots associated with this deployment's lineage
+    def find_snapshots
+      unless @lineage
+        s = @servers.first
+        kind_params = s.transform_parameters(s.parameters)
+        @lineage = kind_params['DB_LINEAGE_NAME'].gsub(/text:/, "")
+      end
+      snapshots = Ec2EbsSnapshot.find_by_cloud_id(@servers.first.cloud_id).select { |n| n.nickname =~ /#{@lineage}.*$/ }
+    end
+
+    # Returns the timestamp of the latest snapshot for testing OPT_DB_RESTORE_TIMESTAMP_OVERRIDE
+    def find_snapshot_timestamp
+      last_snap = behavior(:find_snapshots).last
+      last_snap.tags.detect { |t| t["name"] =~ /timestamp=(\d+)$/ }
+      timestamp = $1
+    end
+
+    # creates a EBS stripe on the server
+    # * server<~Server> the server to create stripe on
+    def create_stripe_volume(server)
+      options = { "EBS_MOUNT_POINT" => "text:#{@mount_point}",
+              "EBS_STRIPE_COUNT" => "text:#{@stripe_count}",
+              "EBS_TOTAL_VOLUME_GROUP_SIZE" => "text:#{@volume_size}",
+              "EBS_LINEAGE" => "text:#{@lineage}" }
+      audit = server.run_executable(@scripts_to_run['create_stripe'], options)
+      audit.wait_for_completed
+    end
+
+    # * server<~Server> the server to restore to
+    def restore_from_backup(server,force)
+      options = { "EBS_MOUNT_POINT" => "text:#{@mount_point}",
+              "OPT_DB_FORCE_RESTORE" => "text:#{force}",
+              "EBS_LINEAGE" => "text:#{@lineage}" }
+      audit = server.run_executable(@scripts_to_run['restore'], options)
+      audit.wait_for_completed
+    end
+
+    # * server<~Server> the server to restore to
+    def restore_and_grow(server,new_size,force)
+      options = { "EBS_MOUNT_POINT" => "text:#{@mount_point}",
+              "EBS_TOTAL_VOLUME_GROUP_SIZE" => "text:#{new_size}",
+              "OPT_DB_FORCE_RESTORE" => "text:#{force}",
+              "EBS_LINEAGE" => "text:#{@lineage}" }
+      audit = server.run_executable(@scripts_to_run['grow_volume'], options)
+      audit.wait_for_completed
+    end
+
+    # Verify that the volume has special data on it.
+    def test_volume_data(server)
+      server.spot_check_command("test -f #{@mount_point}/data.txt")
+    end
+
+    # Verify that the volume is the expected size
+    def test_volume_size(server,expected_size)
+      puts "Testing with: #{@mount_point} #{expected_size}"
+puts "THIS DOES NOT WORK - cause of rounding errors during volume size determination, FS overhead  and df's output"
+puts "Need to query the volumes attached to the server and verify that they #{expected_size}/#{@stripe_count}"
+puts "Check that the server's volumes are #{expected_size}"
+#      server.spot_check_command("df -kh | awk -F\" \" -v -v size=#{expected_size}G '/#{@mount_point}/ {exit $2!=size}'")
+    end
+
+    # Writes data to the EBS volume so snapshot restores can be verified
+    # Not sure what to write...... Maybe pass a string to write to a file??..
+    def populate_volume(server)
+       server.spot_check_command(" echo \"blah blah blah\" > #{@mount_point}/data.txt")
+    end
+
+    # * server<~Server> the server to terminate
+    def terminate_server(server)
+      options = { "EBS_MOUNT_POINT" => "text:#{@mount_point}",
+              "EBS_TERMINATE_SAFETY" => "text:off" }
+      audit = server.run_executable(@scripts_to_run['terminate'], options)
+      audit.wait_for_completed
+    end
+
+    # Use the termination script to stop all the servers (this cleans up the volumes)
+    def stop_all
+      @servers.each do |s|
+        terminate_server(s) if s.state == 'operational' || s.state == 'stranded'
+      end
+      @servers.each { |s| s.wait_for_state("stopped") }
+      # unset dns in our local cached copy..
+      @servers.each { |s| s.params['dns-name'] = nil }
+    end
+    def test_restore_grow
+      grow_to_size=100
+      behavior(:restore_and_grow, s_three, grow_to_size, false)
+      behavior(:test_volume_data, s_three)
+      behavior(:test_volume_size, s_three, grow_to_size)
+    end
+
+    def test_restore
+      behavior(:restore_from_backup, s_two, false)
+      behavior(:test_volume_data, s_two)
+    end
+
+    def create_backup
+      behavior(:run_script, "backup", s_one)
+      behavior(:wait_for_snapshots)
+    end
+
+  end
+end