virtualmonkey 0.0.1

data/features/start_only.rb

@@ -0,0 +1,26 @@
+#@base
+#
+# Feature: Base Server Test
+#   Tests the base server functions
+#
+# Scenario: base server test
+#
+# Given A simple deployment
+  @runner = VirtualMonkey::SimpleRunner.new(ENV['DEPLOYMENT'])
+
+# Then I should stop the servers
+  @runner.behavior(:stop_all)
+
+# Then I should launch all servers
+  @runner.behavior(:launch_all)
+
+# Then I should wait for the state of "all" servers to be "operational"
+  @runner.behavior(:wait_for_all, "operational")
+
+# Then I should reboot the servers
+  @runner.behavior(:reboot_all)
+
+# Then I should wait for the state of "all" servers to be "operational"
+  @runner.behavior(:wait_for_all, "operational")
+
+

data/features/tomcat6.rb

@@ -0,0 +1,54 @@
+#@lb_test
+#
+#Feature: Tomcat6 Server Templates
+#  Tests the Tomcat6 servers
+#
+#Scenario: Tomcat6 server test
+#
+# Given A frontend with application servers deployment
+  @runner = VirtualMonkey::FeAppRunner.new(ENV['DEPLOYMENT'])
+
+# Then I should stop the servers
+  @runner.behavior(:stop_all)
+
+# Then I should set a variation for connecting to shared database host
+  @runner.set_var(:set_master_db_dnsname)
+
+# When I launch the "Front End" servers
+  @runner.behavior(:launch_set, "Front End")
+
+# Then I should wait for the state of "Front End" servers to be "booting"
+  @runner.behavior(:wait_for_set, "Front End", "booting")
+
+# Then I should wait for the state of "Front End" servers to be "operational"
+  @runner.behavior(:wait_for_set, "Front End", "operational")
+
+# Then I should set a variation LB_HOSTNAME
+  @runner.set_var(:set_lb_hostname)
+
+# When I launch the "App Server" servers
+  @runner.behavior(:launch_set, "App Server")
+
+# Then I should wait for the state of "App Server" servers to be "booting"
+  @runner.behavior(:wait_for_set, "App Server", "booting")
+
+# Then I should wait for the state of "App Server" servers to be "operational"
+  @runner.behavior(:wait_for_set, "App Server", "operational")
+
+# Then I should cross connect the frontends
+  @runner.behavior(:cross_connect_frontends)
+
+# Then I should run unified application checks
+  @runner.behavior(:run_unified_application_checks, @runner.send(:app_servers))
+
+# Then I should run frontend checks
+  @runner.behavior(:frontend_checks)
+
+# Then I should run log rotation checks
+  @runner.behavior(:log_rotation_checks)
+
+# Then I should test reboot operations on the deployment
+  @runner.behavior(:run_reboot_operations)
+
+# Then I should check that monitoring is enabled
+  @runner.behavior(:check_monitoring)

data/lib/virtualmonkey.rb

@@ -0,0 +1,28 @@
+require 'rest_connection'
+require 'fog'
+require 'fileutils'
+require 'virtualmonkey/unified_application'
+require 'virtualmonkey/file_locations'
+require 'virtualmonkey/deployment_monk'
+require 'virtualmonkey/cuke_monk'
+require 'virtualmonkey/shared_dns'
+require 'virtualmonkey/test_case_interface'
+require 'virtualmonkey/deployment_runner'
+require 'virtualmonkey/ebs'
+require 'virtualmonkey/ebs_runner'
+require 'virtualmonkey/mysql'
+require 'virtualmonkey/mysql_runner'
+require 'virtualmonkey/mysql_toolbox_runner'
+require 'virtualmonkey/application'
+require 'virtualmonkey/frontend'
+require 'virtualmonkey/application_frontend'
+require 'virtualmonkey/simple'
+require 'virtualmonkey/fe_app_runner'
+require 'virtualmonkey/php_aio_trial_chef_runner'
+require 'virtualmonkey/rails_aio_developer_chef_runner'
+require 'virtualmonkey/php_chef_runner'
+require 'virtualmonkey/simple_runner'
+require 'virtualmonkey/lamp_runner'
+require 'virtualmonkey/elb_runner'
+require 'virtualmonkey/command'
+require 'virtualmonkey/patch_runner'

data/lib/virtualmonkey/application.rb

@@ -0,0 +1,75 @@
+require 'timeout'
+
+module VirtualMonkey
+  module Application
+
+    # returns an Array of the App Servers in the deployment
+    def app_servers
+      ret = @servers.select { |s| s.nickname =~ /App Server/ }
+      raise "No app servers in deployment" unless ret.length > 0
+      ret
+    end
+    
+    # sets LB_HOSTNAME on the deployment using the private dns of the fe_servers
+    def set_lb_hostname
+      @deployment.set_input("LB_HOSTNAME", get_lb_hostname_input)
+    end
+
+    # returns true if the http response contains the expected_string
+    # * url<~String> url to perform http request
+    # * expected_string<~String> regex compatible string used to match against the response output
+    def test_http_response(expected_string, url, port)
+      cmd = "curl -s #{url} 2> /dev/null "
+      puts cmd
+      timeout=300
+      begin
+        status = Timeout::timeout(timeout) do
+          while true
+            response = `#{cmd}`
+            puts response 
+            break if response.include?(expected_string)
+            puts "Retrying..."
+            sleep 5
+          end
+        end
+      rescue Timeout::Error => e
+        raise "ERROR: Query failed after #{timeout/60} minutes."
+      end
+    end
+    
+    def run_rails_demo_application_checks(run_on=@servers,port=80)
+      run_on.each do |server|
+        url_base = "#{server.dns_name}:#{port}"
+        test_http_response("Mephisto", url_base, port) 
+      end
+    end
+
+    def lookup_scripts
+      @scripts_to_run = {}
+      st = ServerTemplate.find(@servers.first.server_template_href)
+      @scripts_to_run['apache_restart'] = st.executables.detect { |ex| ex.name =~  /WEB apache \(re\)start v2/i }
+    end 
+
+    # Assumes the host machine is EC2, uses the meta-data to grab the IP address of this
+    # 'tester server' eg. used for the input variation MASTER_DB_DNSNAME
+    def get_tester_ip_addr
+      if File.exists?("/var/spool/ec2/meta-data.rb")
+        require "/var/spool/ec2/meta-data-cache" 
+      else
+        ENV['EC2_PUBLIC_HOSTNAME'] = "127.0.0.1"
+      end
+      my_ip_input = "text:" 
+      my_ip_input += ENV['EC2_PUBLIC_HOSTNAME']
+      my_ip_input
+    end
+
+    # Run spot checks for APP servers in the deployment
+    def run_app_tests
+    end
+
+    # Special startup sequence for an APP deployment
+    def startup
+    end
+
+  end
+end

data/lib/virtualmonkey/application_frontend.rb

@@ -0,0 +1,42 @@
+module VirtualMonkey
+  module ApplicationFrontend
+    include VirtualMonkey::Application
+    include VirtualMonkey::Frontend
+    include VirtualMonkey::UnifiedApplication
+    
+    # a custom startup sequence is required for fe/app deployments (inputs workaround)
+    def startup_sequence
+      fe_servers.each { |s| s.start }
+      fe_servers.each { |s| s.wait_for_operational_with_dns }
+      
+      set_lb_hostname
+
+      app_servers.each { |s| s.start }
+      app_servers.each { |s| s.wait_for_operational_with_dns }
+    end
+
+    def run_reboot_operations
+      behavior(:reboot_all, true)
+      behavior(:run_reboot_checks)
+    end
+
+    def run_reboot_checks
+      behavior(:run_unified_application_checks, fe_servers, 80)
+      behavior(:run_unified_application_checks, app_servers)
+    end
+    
+    def log_rotation_checks
+      # this works for php, TODO: rails
+      #app_servers.each do |server|
+      #  force_log_rotation(server)
+      #  log_check(server,"/mnt/log/#{server.apache_str}/access.log.1")
+      #end
+
+      fe_servers.each do |server|
+        behavior(:force_log_rotation, server)
+        behavior(:log_check, server, "/mnt/log/#{server.apache_str}/haproxy.log.1")
+      end
+    end
+    
+  end
+end

data/lib/virtualmonkey/command.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'trollop'
+require 'highline/import'
+require 'virtualmonkey/command/create'
+require 'virtualmonkey/command/destroy'
+require 'virtualmonkey/command/run'
+require 'virtualmonkey/command/list'
+require 'virtualmonkey/command/troop'
+require 'virtualmonkey/command/clone'
+
+module VirtualMonkey
+  module Command
+    # Parses the initial command string, removing it from ARGV, then runs command.
+    def self.go
+      command = ARGV.shift
+      case command
+        when "create"
+          VirtualMonkey::Command.create
+        when "destroy"
+          VirtualMonkey::Command.destroy
+        when "run"
+          VirtualMonkey::Command.run
+        when "list"
+          VirtualMonkey::Command.list
+        when "troop"
+          VirtualMonkey::Command.troop
+        when "clone"
+          VirtualMonkey::Command.clone
+        when "help" || "--help" || "-h"
+          puts "Help usage: monkey <command> --help"
+          puts "Valid commands for monkey: create, destroy, list, run, troop, clone or help"
+        else
+          STDERR.puts "Invalid command #{command}: You need to specify a command for monkey: create, destroy, list, run, troop, clone or help\n"
+          exit(1)
+      end
+    end
+  end
+end

data/lib/virtualmonkey/command/clone.rb

@@ -0,0 +1,50 @@
+module VirtualMonkey
+  module Command
+
+# monkey clone --deployment name --feature testcase.rb --breakpoint 4 --copies 7
+    def self.clone
+      options = Trollop::options do
+        opt :deployment, "regex string to use for matching deployment", :type => :string, :short => '-d', :required => true
+        opt :feature, "path to feature(s) to run against the deployments", :type => :string
+        opt :breakpoint, "feature file line to stop at", :type => :integer, :short => '-b'
+        opt :copies, "number of copies to make (default is 1)", :type => :integer, :short => '-c'
+      end
+
+      options[:copies] = 1 unless options[:copies] > 1
+      options[:no_resume] = true
+      dm = DeploymentMonk.new(options[:deployment])
+      if dm.deployments.length > 1
+        raise "FATAL: Ambiguous Regex; more than one deployment matched /#{options[:deployment]}/"
+      elsif dm.deployments.length < 1
+        raise "FATAL: Ambiguous Regex; no deployment matched /#{options[:deployment]}/"
+      end
+      origin = dm.deployments.first
+      do_these ||= []
+      # clone deployment
+      for i in 1 .. options[:copies]
+        new_deploy = origin.clone
+        new_deploy.nickname = "#{origin.nickname}-clone-#{i}"
+        new_deploy.save
+        do_these << new_deploy
+      end
+
+      # run to breakpoint
+      if options[:feature]
+        EM.run {
+          cm = CukeMonk.new
+          cm.options = options
+          do_these.each do |deploy|
+            cm.run_test(deploy, options[:feature])
+          end
+
+          watch = EM.add_periodic_timer(10) {
+            if cm.all_done?
+              watch.cancel
+            end
+            cm.watch_and_report
+          }
+        }
+      end
+    end
+  end
+end

data/lib/virtualmonkey/command/create.rb

@@ -0,0 +1,21 @@
+module VirtualMonkey
+  module Command
+
+# monkey create --server_template_ids 123,123 --common_inputs blah.json --feature simple.feature --tag unique_name --TBD:filter?
+    def self.create
+      options = Trollop::options do
+        opt :server_template_ids, "ServerTemplate ids to use for creating the deployment.  Use one ID per server that you would like to be in the deployment.  Accepts space separated integers, or one argument per id. Eg. -s 23747 23747", :type => :integers, :required => true, :short => '-s'
+        opt :common_inputs, "Paths to common input json files to load and set on all deployments.  Accepts space separated pathnames or one argument per pathname.  Eg. -c config/mysql_inputs.json -c config/other_inputs.json", :type => :strings, :required => true, :short => '-c'
+        opt :tag, "Tag to use as nickname prefix for all deployments.", :type => :string, :required => true, :short => '-t'
+        opt :cloud_variables, "Path to json file containing common inputs and variables per cloud. See config/cloud_variables.json.example", :type => :string, :required => true, :short => '-v'
+        opt :no_spot, "Do not use spot instances"
+      end
+      @dm = DeploymentMonk.new(options[:tag], options[:server_template_ids])
+      @dm.variables_for_cloud = JSON::parse(IO.read(options[:cloud_variables]))
+      options[:common_inputs].each do |cipath|
+        @dm.load_common_inputs(cipath)
+      end  
+      @dm.generate_variations(options)
+    end
+  end
+end

data/lib/virtualmonkey/command/destroy.rb

@@ -0,0 +1,51 @@
+module VirtualMonkey
+  module Command
+  
+# monkey destroy --tag unique_tag
+    def self.destroy
+      options = Trollop::options do
+        opt :tag, "Tag to match prefix of the deployments to destroy.", :type => :string, :required => true, :short => '-t'
+        opt :terminate, "Terminate using the specified runner", :type => :string, :required => true, :short => "-r"
+        opt :no_delete, "only terminate, no deletion."
+        opt :yes, "Turn off confirmation for destroy operation"
+      end
+      begin
+        eval("VirtualMonkey::#{options[:terminate]}.new('fgasvgreng243o520sdvnsals')") if options[:terminate]
+      rescue Exception => e
+        raise e unless e.message =~ /Could not find a deployment named/
+        options[:terminate] = "SimpleRunner" if options[:terminate]
+      end
+      @dm = DeploymentMonk.new(options[:tag])
+#      nicks = @dm.deployments.map &:nickname
+      nicks = @dm.deployments.map { |d| d.nickname }
+      nicks.each { |n| say n }
+      unless options[:yes]
+        confirm = ask("Really destroy these #{nicks.length} deployments (y/n)?", lambda { |ans| true if (ans =~ /^[y,Y]{1}/) })
+        raise "Aborting." unless confirm
+      end
+
+      global_state_dir = File.join(File.dirname(__FILE__), "..", "..", "..", "test_states")
+      @dm.deployments.each do |deploy|
+        @runner = eval("VirtualMonkey::#{options[:terminate]}.new(deploy.nickname)")
+        @runner.behavior(:stop_all, false)
+        state_dir = File.join(global_state_dir, deploy.nickname)
+        if File.directory?(state_dir)
+          puts "Deleting state files for #{deploy.nickname}..."
+          Dir.new(state_dir).each do |state_file|
+            if File.extname(state_file) =~ /((rb)|(feature))/
+              File.delete(File.join(state_dir, state_file))
+            end
+          end
+          Dir.rmdir(state_dir)
+        end
+        if @runner.respond_to?(:release_dns) and not options[:no_delete]
+          @runner.behavior(:release_dns)
+        end
+      end
+
+      @dm.destroy_all unless options[:no_delete]
+      say "monkey done."
+    end
+
+  end
+end

data/lib/virtualmonkey/command/list.rb

@@ -0,0 +1,10 @@
+module VirtualMonkey
+  module Command
+    def self.list
+      options = Trollop::options do
+        opt :tags, "List deployment set tags", :type => :string, :required => true
+      end
+      DeploymentMonk.new(options[:tags]).deployments.each { |d| puts d.nickname }
+    end
+  end 
+end

data/lib/virtualmonkey/command/run.rb

@@ -0,0 +1,76 @@
+require 'eventmachine'
+module VirtualMonkey
+  module Command
+  
+# trollop supports Chronic for human readable dates. use with run command for delayed run?
+
+# monkey run --feature --tag --only <regex to match on deploy nickname>
+    def self.run
+      options = Trollop::options do
+        opt :feature, "path to feature(s) to run against the deployments", :type => :string, :required => true
+        opt :breakpoint, "feature file line to stop at", :type => :integer, :short => '-b'
+        opt :tag, "Tag to match prefix of the deployments.", :type => :string, :required => true, :short => "-t"
+        opt :only, "regex string to use for subselection matching on deployments.  Eg. --only x86_64", :type => :string
+        opt :terminate, "Terminate using the specified runner if feature successfully completes. (No destroy)", :type => :string, :short => "-r"
+        opt :no_resume, "Do not use current test-in-progress, start from scratch", :short => "-n"
+        opt :yes, "Turn off confirmation", :short => "-y"
+      end
+
+      global_state_dir = File.join(File.dirname(__FILE__), "..", "..", "..", "test_states")
+      begin
+        eval("VirtualMonkey::#{options[:terminate]}.new('fgasvgreng243o520sdvnsals')") if options[:terminate]
+      rescue Exception => e
+        raise e unless e.message =~ /Could not find a deployment named/
+        options[:terminate] = "SimpleRunner" if options[:terminate]
+      end
+      EM.run {
+        cm = CukeMonk.new
+        dm = DeploymentMonk.new(options[:tag])
+        if options[:only]
+          do_these = dm.deployments.select { |d| d.nickname =~ /#{options[:only]}/ }
+        else
+          do_these = dm.deployments
+        end
+
+        unless options[:no_resume]
+          temp = do_these.select do |d|
+            File.exist?(File.join(global_state_dir, d.nickname, File.basename(options[:feature])))
+          end
+          do_these = temp if temp.length > 0
+        end
+
+        cm.options = options
+        raise "No deployments matched!" unless do_these.length > 0
+        do_these.each { |d| say d.nickname }
+
+        unless options[:yes]
+          confirm = ask("Run tests on these #{do_these.length} deployments (y/n)?", lambda { |ans| true if (ans =~ /^[y,Y]{1}/) })
+          raise "Aborting." unless confirm
+        end
+
+        remaining_jobs = cm.jobs.dup
+        do_these.each do |deploy|
+          cm.run_test(deploy, options[:feature])
+        end
+
+        watch = EM.add_periodic_timer(10) {
+          cm.watch_and_report
+          if cm.all_done?
+            watch.cancel
+          end
+          if options[:terminate]
+            remaining_jobs.each do |job|
+              if job.status == 0
+                @runner = eval("VirtualMonkey::#{options[:terminate]}.new(job.deployment.nickname)")
+                puts "destroying successful deployment: #{@runner.deployment.nickname}"
+                @runner.behavior(:stop_all, false)
+                remaining_jobs.delete(job)
+              end
+            end
+          end
+        }
+
+      }
+    end
+  end
+end