cfn_manage 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3c9938fbe475e8c39793a88a94253a0ed956ed08
-  data.tar.gz: 2d939e4715fff3bd538e35e39bf5b0d91b70d3cd
+SHA256:
+  metadata.gz: 069e9448ce6f6efafd546df321dd4dbb52503f73586d23f718dc10c211e0de36
+  data.tar.gz: 9cb836b6ea9b0071743386dd7fe030c6c3f013b1be2270671b1df31722b0a2ea
 SHA512:
-  metadata.gz: ad06bdd60a71c78474bcb188abf2b08677f6e0fca11431f751f466c0930f1cc72f6d2a98ef7a20f59675ff6eb6018f2d4dc44873d66731b14b0760165937c705
-  data.tar.gz: 0a634fcb6c0f229ba3ce36c85c33f84f68618f390111f96b0c2dc38cb1e420d4663c45c959bd557c6462b49aa897a3b9439c090f57b53727d8f389bbdf0ec229
+  metadata.gz: a8e2ae95ea5bb3600a46cc637a32f969831190ea2b3cf3d8553a7ee9e4f726ad6aab360f7cc389f08e2fd1df7a1c507cde5e7a9142b79179359590c908048424
+  data.tar.gz: 7d2dcc801ce2c239519fcc8f410b4f2395f56963c8b89e8b2d169131b00f588126b1aa65ccb5a1bff91d6f4434f562ee535b3609e4f6c2585fed195f4ff338c2

data/bin/cfn_manage

@@ -1,2 +1,158 @@
 #!/usr/bin/env ruby
-require_relative('./cfn_manage')
+require 'optparse'
+require 'cfn_manage/version'
+require 'cfn_manage/cf_common'
+require 'cfn_manage/cf_start_stop_environment'
+require 'logger'
+
+# exit with usage information
+def print_usage_exit(code)
+  STDERR.puts(File.open("#{File.expand_path(File.dirname(__FILE__))}/usage.txt").read)
+  exit code
+end
+
+# global options
+$options = {}
+$options['SOURCE_BUCKET'] = ENV['SOURCE_BUCKET']
+$options['AWS_ASSUME_ROLE'] = ENV['AWS_ASSUME_ROLE']
+
+# global logger
+$log = Logger.new(STDOUT)
+
+# always flush output
+STDOUT.sync = true
+
+# parse command line options
+OptionParser.new do |opts|
+
+  Version = CfnManage::VERSION
+
+  opts.banner = 'Usage: cfn_manage [command] [options]'
+
+  opts.on('--source-bucket [BUCKET]') do |bucket|
+    $options['SOURCE_BUCKET'] = bucket
+    ENV['SOURCE_BUCKET'] = bucket
+  end
+
+  opts.on('--aws-role [ROLE]') do |role|
+    ENV['AWS_ASSUME_ROLE'] = role
+  end
+
+  opts.on('--stack-name [STACK_NAME]') do |stack|
+    $options['STACK_NAME'] = stack
+  end
+
+  opts.on('--asg-name [ASG]') do |asg|
+    $options['ASG'] = asg
+  end
+
+  opts.on('--rds-instance-id [RDS_INSTANCE_ID]') do |rds|
+    $options['RDS_INSTANCE_ID'] = rds
+  end
+
+  opts.on('--aurora-cluster-id [AURORA_CLUSTER_ID]') do |cluster|
+    $options['AURORA_CLUSTER_ID'] = cluster
+  end
+
+  opts.on('--ec2-instance-id [EC2_INSTANCE_ID]') do |ec2|
+    $options['EC2_INSTANCE_ID'] = ec2
+  end
+
+  opts.on('--spot-fleet-id [SPOT_FLEET]') do |spot|
+    $options['SPOT_FLEET'] = spot
+  end
+
+  opts.on('--ecs-cluster [ECS_CLUSTER]') do |ecs|
+    $options['ECS_CLUSTER'] = ecs
+  end
+
+  opts.on('--alarm [ALARM]') do |alarm|
+    $options['ALARM'] = alarm
+  end
+
+  opts.on('-r [AWS_REGION]', '--region [AWS_REGION]') do |region|
+    ENV['AWS_REGION'] = region
+  end
+
+  opts.on('-p [AWS_PROFILE]', '--profile [AWS_PROFILE]') do |profile|
+    ENV['CFN_AWS_PROFILE'] = profile
+  end
+
+  opts.on('--dry-run') do
+    ENV['DRY_RUN'] = '1'
+  end
+
+  opts.on('--continue-on-error') do
+    ENV['CFN_CONTINUE_ON_ERROR'] = '1'
+  end
+
+  opts.on('--wait-async') do
+    ENV['WAIT_ASYNC'] = '1'
+    ENV['SKIP_WAIT'] = '1'
+  end
+
+  opts.on('--skip_wait') do
+    ENV['SKIP_WAIT'] = '1'
+  end
+
+end.parse!
+
+command = ARGV[0]
+
+if command.nil?
+  print_usage_exit(-1)
+end
+
+# execute action based on command
+case command
+  when 'help'
+    print_usage_exit(0)
+  # asg commands
+  when 'stop-asg'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ASG'],'AWS::AutoScaling::AutoScalingGroup')
+  when 'start-asg'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ASG'],'AWS::AutoScaling::AutoScalingGroup')
+
+  # rds commands
+  when 'stop-rds'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['RDS_INSTANCE_ID'],'AWS::RDS::DBInstance')
+  when 'start-rds'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['RDS_INSTANCE_ID'],'AWS::RDS::DBInstance')
+
+  # aurora cluster commands
+  when 'stop-aurora-cluster'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['AURORA_CLUSTER_ID'],'AWS::RDS::DBCluster')
+  when 'start-aurora-cluster'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['AURORA_CLUSTER_ID'],'AWS::RDS::DBCluster')
+
+  # ec2 instance
+  when 'stop-ec2'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['EC2_INSTANCE_ID'],'AWS::EC2::Instance')
+  when 'start-ec2'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['EC2_INSTANCE_ID'],'AWS::EC2::Instance')
+
+  # spot fleet
+  when 'stop-spot-fleet'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['SPOT_FLEET'],'AWS::EC2::SpotFleet')
+  when 'start-spot-fleet'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['SPOT_FLEET'],'AWS::EC2::SpotFleet')
+
+    # spot fleet
+  when 'stop-ecs-cluster'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ECS_CLUSTER'],'AWS::ECS::Cluster')
+  when 'start-ecs-cluster'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ECS_CLUSTER'],'AWS::ECS::Cluster')
+
+
+  # cloudwatch alarm
+  when 'disable-alarm'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ALARM'],'AWS::CloudWatch::Alarm')
+  when 'enable-alarm'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_resource($options['ALARM'],'AWS::CloudWatch::Alarm')
+
+  # stack commands
+  when 'stop-environment'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().stop_environment($options['STACK_NAME'])
+  when 'start-environment'
+    CfnManage::CloudFormation::EnvironmentRunStop.new().start_environment($options['STACK_NAME'])
+end

data/bin/usage.txt

@@ -2,6 +2,10 @@ Usage: cfn_manage [command] [options]
 
 Commands:
 
+cfn_manage help
+
+cfn_manage version
+
 cfn_manage stop-environment --stack-name [STACK_NAME]
 
 cfn_manage start-environment --stack-name [STACK_NAME]
@@ -18,7 +22,23 @@ cfn_manage stop-aurora-cluster --aurora-cluster-id [AURORA_CLUSTER_ID]
 
 cfn_manage start-aurora-cluster --aurora-cluster-id [AURORA_CLUSTER_ID]
 
-General options
+cfn_manage stop-ec2 --ec2-instance-id [EC2_INSTANCE_ID]
+
+cfn_manage start-ec2 --ec2-instance-id [EC2_INSTANCE_ID]
+
+cfn_manage stop-spot-fleet --spot-fleet [SPOT_FLEET]
+
+cfn_manage start-spot-fleet --spot-fleet [SPOT_FLEET]
+
+cfn_manage stop-ecs-cluster --ecs-cluster [ECS_CLUSTER]
+
+cfn_manage start-ecs-cluster --ecs-cluster [ECS_CLUSTER]
+
+cfn_manage disable-alarm --alarm [ALARM]
+
+cfn_manage enable-alarm --alarm [ALARM]
+
+General options:
 
 --source-bucket [BUCKET]
 
@@ -48,5 +68,15 @@ General options
 
     Applicable only to [start|stop-environment] commands. If there is problem with stopping a resource,
     (e.g. cloudformation stack not being synced or manual resource deletion) script will continue it's
-    operation. By defult script stops when there is problem with starting/stopping resource, and expects
+    operation. By default script stops when there is problem with starting/stopping resource, and expects
     manual intervention to fix the root cause for failure.
+
+--skip-wait
+
+    Skips waiting for resources to achieve stopped or started states.
+
+--wait-async
+
+  Default wait action is to wait for each individual resource to be stopped and started before continuing.
+  This will enabled waiting for resources in groups based on priority. Option only useful when used with
+  start-environment and stop-environment commands.

data/lib/cfn_manage.rb

@@ -0,0 +1 @@
+module CfnManage; end

data/lib/{alarm_start_stop_handler.rb → cfn_manage/alarm_start_stop_handler.rb}

@@ -1,12 +1,12 @@
-require_relative '../lib/aws_credentials'
+require 'cfn_manage/aws_credentials'
 
-module Base2
+module CfnManage
 
   class AlarmStartStopHandler
 
     def initialize(alarm_name)
       @alarm_id = alarm_name
-      credentials = Base2::AWSCredentials.get_session_credentials("startstopalarm_#{@asg_name}")
+      credentials = CfnManage::AWSCredentials.get_session_credentials("startstopalarm_#{@asg_name}")
       @cwclient = Aws::CloudWatch::Client.new(retry_limit: 20)
       if credentials != nil
         @cwclient = Aws::CloudWatch::Client.new(credentials: credentials, retry_limit: 20)
@@ -34,6 +34,11 @@ module Base2
       @alarm.disable_actions({})
       return {}
     end
+
+    def wait(wait_states=[])
+      $log.debug("Not waiting for alarm #{@alarm_id}")
+    end
+
   end
 
-end
+end

data/lib/{asg_start_stop_handler.rb → cfn_manage/asg_start_stop_handler.rb}

@@ -1,13 +1,13 @@
-require_relative '../lib/aws_credentials'
+require 'cfn_manage/aws_credentials'
 
-module Base2
+module CfnManage
 
   class AsgStartStopHandler
 
-    def initialize(asg_id)
+    def initialize(asg_id, skip_wait)
       @asg_name = asg_id
-
-      credentials = Base2::AWSCredentials.get_session_credentials("stopasg_#{@asg_name}")
+      @skip_wait = skip_wait
+      credentials = CfnManage::AWSCredentials.get_session_credentials("stopasg_#{@asg_name}")
       @asg_client = Aws::AutoScaling::Client.new(retry_limit: 20)
       if credentials != nil
         @asg_client = Aws::AutoScaling::Client.new(credentials: credentials, retry_limit: 20)
@@ -66,5 +66,9 @@ module Base2
       })
     end
 
+    def wait(wait_states=[])
+      $log.debug("Not waiting for ASG #{@asg_name}")
+    end
+
   end
-end
+end

data/lib/{aurora_cluster_start_stop_handler.rb → cfn_manage/aurora_cluster_start_stop_handler.rb}

@@ -1,13 +1,13 @@
-require_relative '../lib/aws_credentials'
+require 'cfn_manage/aws_credentials'
 
-module Base2
+module CfnManage
 
   class AuroraClusterStartStopHandler
 
-    def initialize(cluster_id)
+    def initialize(cluster_id, skip_wait)
       @cluster_id = cluster_id
-
-      credentials = Base2::AWSCredentials.get_session_credentials("startstopcluster_#{cluster_id}")
+      @skip_wait = skip_wait
+      credentials = CfnManage::AWSCredentials.get_session_credentials("startstopcluster_#{cluster_id}")
       @rds_client = Aws::RDS::Client.new(retry_limit: 20)
       if credentials != nil
         @rds_client = Aws::RDS::Client.new(credentials: credentials, retry_limit: 20)
@@ -27,10 +27,11 @@ module Base2
       if @rds_cluster.status == 'stopped'
         $log.info("Starting Aurora cluster #{@cluster_id}")
         @rds_client.start_db_cluster({ db_cluster_identifier: @cluster_id })
-
-        # wait cluster to become available
-        $log.info("Waiting Aurora cluster to become available #{@cluster_id}")
-        wait_rds_cluster_states( %w(starting available))
+        unless @skip_wait
+          # wait cluster to become available
+          $log.info("Waiting Aurora cluster to become available #{@cluster_id}")
+          wait('available')
+        end
       else
         $log.info("Aurora Cluster #{@cluster_id} is not in a stopped state. State: #{@rds_cluster.status}")
       end
@@ -46,41 +47,39 @@ module Base2
         $log.info("Aurora Cluster #{@cluster_id} is not in a available state. State: #{@rds_cluster.status}")
         return {}
       end
-
       # stop rds cluster and wait for it to be fully stopped
       $log.info("Stopping aurora cluster #{@cluster_id}")
       @rds_client.stop_db_cluster({ db_cluster_identifier: @cluster_id })
-      $log.info("Waiting aurora cluster to be stopped #{@cluster_id}")
-      wait_rds_cluster_states(%w(stopping stopped))
-
+      unless @skip_wait
+        $log.info("Waiting aurora cluster to be stopped #{@cluster_id}")
+        wait('stopped')
+      end
       return {}
     end
 
-    def wait_rds_cluster_states(wait_states=[])
-      wait_states.each do |state|
-        # reached state must be steady, at least a minute.
-        state_count = 0
-        steady_count = 4
-        attempts = 0
-        rds = Aws::RDS::Resource.new(client: @rds_client)
-        until attempts == (max_attempts = 60*6) do
-          cluster = rds.db_cluster(@cluster_id)
-          $log.info("Aurora Cluster #{cluster.db_cluster_identifier} state: #{cluster.status}, waiting for #{state}")
-
-          if cluster.status == "#{state}"
-            state_count = state_count + 1
-            $log.info("#{state_count}/#{steady_count}")
-          else
-            state_count = 0
-          end
-          break if state_count == steady_count
-          attempts = attempts + 1
-          sleep(15)
+    def wait(completed_state)
+      # reached state must be steady, at least a minute.
+      state_count = 0
+      steady_count = 4
+      attempts = 0
+      rds = Aws::RDS::Resource.new(client: @rds_client)
+      until attempts == (max_attempts = 60*6) do
+        cluster = rds.db_cluster(@cluster_id)
+        $log.info("Aurora Cluster #{cluster.db_cluster_identifier} state: #{cluster.status}, waiting for #{completed_state}")
+
+        if cluster.status == "#{completed_state}"
+          state_count = state_count + 1
+          $log.info("#{state_count}/#{steady_count}")
+        else
+          state_count = 0
         end
+        break if state_count == steady_count
+        attempts = attempts + 1
+        sleep(15)
+      end
 
-        if attempts == max_attempts
-          $log.error("RDS Aurora Cluster #{@cluster_id} did not enter #{state} state, however continuing operations...")
-        end
+      if attempts == max_attempts
+        $log.error("RDS Aurora Cluster #{@cluster_id} did not enter #{state} state, however continuing operations...")
       end
     end
 

data/lib/{aws_credentials.rb → cfn_manage/aws_credentials.rb}

@@ -1,6 +1,6 @@
 require 'aws-sdk-core'
 
-module Base2
+module CfnManage
 
   class AWSCredentials
 
@@ -33,4 +33,4 @@ module Base2
 
     end
   end
-end
+end

data/lib/{cf_common.rb → cfn_manage/cf_common.rb}

@@ -1,4 +1,4 @@
-module Base2
+module CfnManage
 
   module CloudFormation
 

data/lib/{cf_progress_tracker.rb → cfn_manage/cf_progress_tracker.rb}

@@ -1,7 +1,7 @@
 require 'aws-sdk-cloudformation'
-require_relative './cf_common'
+require 'cfn_manage/cf_common'
 
-module Base2
+module CfnManage
   module CloudFormation
     class ProgressTracker
       @cf_client = nil
@@ -34,17 +34,17 @@ module Base2
         @period_from = period_from
       end
 
-       
+
       def track_single_stack(stack)
         stack_id = stack['stack_id']
         # Default to period_from if first run, take from last run otherwise
         event_from = last_event_times[stack_id] if @last_event_times.key?(stack_id)
         event_from = @period_from unless @last_event_times.key?(stack_id)
-        
+
 
         stack_resources = @cf_client.describe_stack_events(stack_name: stack['stack_id'],)
-        
-        
+
+
       end
 
       def track_progress(_show_only_failures = false)

data/lib/{cf_start_stop_environment.rb → cfn_manage/cf_start_stop_environment.rb}

@@ -5,14 +5,15 @@ require 'aws-sdk-cloudformation'
 require 'aws-sdk-rds'
 require 'aws-sdk-cloudwatch'
 require 'aws-sdk-autoscaling'
+require 'aws-sdk-ecs'
 
-require_relative '../lib/cf_common'
-require_relative '../lib/aws_credentials'
+require 'cfn_manage/cf_common'
+require 'cfn_manage/aws_credentials'
 require 'json'
 require 'yaml'
-require_relative '../lib/start_stop_handler_factory'
+require 'cfn_manage/start_stop_handler_factory'
 
-module Base2
+module CfnManage
   module CloudFormation
     class EnvironmentRunStop
 
@@ -29,6 +30,7 @@ module Base2
           'AWS::AutoScaling::AutoScalingGroup' => '200',
           'AWS::EC2::Instance' => '200',
           'AWS::EC2::SpotFleet' => '200',
+          'AWS::ECS::Cluster' => '250',
           'AWS::CloudWatch::Alarm' => '300'
       }
 
@@ -37,11 +39,13 @@ module Base2
         @s3_client = Aws::S3::Client.new(retry_limit: 20)
         @s3_bucket = ENV['SOURCE_BUCKET']
         @cf_client = Aws::CloudFormation::Client.new(retry_limit: 20)
-        @credentials = Base2::AWSCredentials.get_session_credentials('start_stop_environment')
+        @credentials = CfnManage::AWSCredentials.get_session_credentials('start_stop_environment')
         if not @credentials.nil?
           @cf_client = Aws::CloudFormation::Client.new(credentials: @credentials, retry_limit: 20)
         end
         @dry_run = (ENV.key?('DRY_RUN') and ENV['DRY_RUN'] == '1')
+        @skip_wait = (ENV.key?('SKIP_WAIT') and ENV['SKIP_WAIT'] == '1')
+        @wait_async = (ENV.key?('WAIT_ASYNC') and ENV['WAIT_ASYNC'] == '1')
         @continue_on_error = (ENV.key? 'CFN_CONTINUE_ON_ERROR' and ENV['CFN_CONTINUE_ON_ERROR'] == '1')
       rescue NoMethodError => e
         puts "Got No Method Error on CloudFormation::initialize, this often means that you're missing a AWS_DEFAULT_REGION"
@@ -69,92 +73,131 @@ module Base2
         $log.info("Environment #{stack_name} stopped")
       end
 
-      def start_stop_aurora_cluster(action,cluster_id)
-        start_stop_handler = Base2::StartStopHandlerFactory.get_start_stop_handler(
-            'AWS::RDS::DBCluster',
-            cluster_id
+      def start_resource(resource_id,resource_type)
+        start_stop_handler = CfnManage::StartStopHandlerFactory.get_start_stop_handler(
+            resource_type,
+            resource_id,
+            @skip_wait
         )
         @environment_resources << {
-            id: cluster_id,
-            priority: '100',
+            id: resource_id,
+            priority: @@resource_start_priorities[resource['resource_type']],
             handler: start_stop_handler,
-            type: 'AWS::RDS::DBCluster'
+            type: resource_type
         }
-        do_stop_assets if action == 'stop'
-        do_start_assets if action == 'start'
+        do_start_assets
       end
 
-      def start_stop_rds(action,instance_id)
-        start_stop_handler = Base2::StartStopHandlerFactory.get_start_stop_handler(
-            'AWS::RDS::DBInstance',
-            instance_id
+      def stop_resource(resource_id,resource_type)
+        start_stop_handler = CfnManage::StartStopHandlerFactory.get_start_stop_handler(
+            resource_type,
+            resource_id,
+            @skip_wait
         )
         @environment_resources << {
-            id: instance_id,
-            priority: '100',
+            id: resource_id,
+            priority: @@resource_start_priorities[resource['resource_type']],
             handler: start_stop_handler,
-            type: 'AWS::RDS::DBInstance'
+            type: resource_type
         }
-        do_stop_assets if action == 'stop'
-        do_start_assets if action == 'start'
+        do_stop_assets
       end
 
       def do_stop_assets
         # sort start resource by priority
         @environment_resources = @environment_resources.sort_by { |k| k[:priority] }.reverse
+        environment_resources_by_priority = @environment_resources.partition { |k| k[:priority] }
 
-        @environment_resources.each do |resource|
-          begin
-            $log.info("Stopping resource #{resource[:id]}")
-            # just print out information if running a dry run, otherwise start assets
-            if not @dry_run
-              configuration = resource[:handler].stop()
-              if configuration.class == Hash
-                s3_prefix = "environment_data/resource/#{resource[:id]}"
-                save_item_configuration(s3_prefix, configuration)
+        environment_resources_by_priority.each do |priority|
+          priority.each do |resource|
+            begin
+              $log.info("Stopping resource #{resource[:id]}")
+              # just print out information if running a dry run, otherwise start assets
+              if not @dry_run
+                configuration = resource[:handler].stop()
+                if configuration.class == Hash
+                  s3_prefix = "environment_data/resource/#{resource[:id]}"
+                  save_item_configuration(s3_prefix, configuration)
+                end
+              else
+                $log.info("Dry run enabled, skipping stop start\nFollowing resource would be stopped: #{resource[:id]}")
+                $log.debug("Resource type: #{resource[:type]}\n\n")
+              end
+            rescue => e
+              $log.error("An exception occurred during stop operation against resource #{resource[:id]}")
+              $log.error("#{e.to_s}")
+              $log.error(e.backtrace.join("\n\t"))
+              if not @continue_on_error
+                raise e
               end
-            else
-              $log.info("Dry run enabled, skipping stop start\nFollowing resource would be stopped: #{resource[:id]}")
-              $log.debug("Resource type: #{resource[:type]}\n\n")
             end
-          rescue => e
-            $log.error("An exception occurred during stop operation against resource #{resource[:id]}")
-            $log.error("#{e.to_s}")
-            $log.error(e.backtrace.join("\n\t"))
-            if not @continue_on_error
-              raise e
+          end
+
+          if not @dry_run and @wait_async
+            priority.each do |resource|
+              begin
+                resource[:handler].wait('stopped')
+              rescue => e
+                $log.error("An exception occurred during wait operation against resource #{resource[:id]}")
+                $log.error("#{e.to_s}")
+                $log.error(e.backtrace.join("\n\t"))
+                if not @continue_on_error
+                  raise e
+                end
+              end
             end
           end
+
         end
       end
 
       def do_start_assets
         # sort start resource by priority
         @environment_resources = @environment_resources.sort_by { |k| k[:priority] }
+        environment_resources_by_priority = @environment_resources.partition { |k| k[:priority] }
 
-        @environment_resources.each do |resource|
-          begin
-            $log.info("Starting resource #{resource[:id]}")
-            # just print out information if running a dry run, otherwise start assets
-            if not @dry_run
-              # read configuration
-              s3_prefix = "environment_data/resource/#{resource[:id]}"
-              configuration = get_object_configuration(s3_prefix)
-
-              # start
-              resource[:handler].start(configuration)
-            else
-              $log.info("Dry run enabled, skipping actual start\nFollowing resource would be started: #{resource[:id]}")
-              $log.debug("Resource type: #{resource[:type]}\n\n")
+        environment_resources_by_priority.each do |priority|
+          priority.each do |resource|
+            begin
+              $log.info("Starting resource #{resource[:id]}")
+              # just print out information if running a dry run, otherwise start assets
+              if not @dry_run
+                # read configuration
+                s3_prefix = "environment_data/resource/#{resource[:id]}"
+                configuration = get_object_configuration(s3_prefix)
+
+                # start
+                resource[:handler].start(configuration)
+              else
+                $log.info("Dry run enabled, skipping actual start\nFollowing resource would be started: #{resource[:id]}")
+                $log.debug("Resource type: #{resource[:type]}\n\n")
+              end
+            rescue => e
+                $log.error("An exception occurred during start operation against resource #{resource[:id]}")
+                $log.error("#{e.to_s}")
+                $log.error(e.backtrace.join("\n\t"))
+                if not @continue_on_error
+                  raise e
+                end
             end
-          rescue => e
-              $log.error("An exception occurred during start operation against resource #{resource[:id]}")
-              $log.error("#{e.to_s}")
-              $log.error(e.backtrace.join("\n\t"))
-              if not @continue_on_error
-                raise e
+          end
+
+          if not @dry_run and @wait_async
+            priority.each do |resource|
+              begin
+                resource[:handler].wait('available')
+              rescue => e
+                $log.error("An exception occurred during wait operation against resource #{resource[:id]}")
+                $log.error("#{e.to_s}")
+                $log.error(e.backtrace.join("\n\t"))
+                if not @continue_on_error
+                  raise e
+                end
               end
+            end
+
           end
+
         end
       end
 
@@ -163,9 +206,10 @@ module Base2
         resrouces['stack_resources'].each do |resource|
           start_stop_handler = nil
           begin
-            start_stop_handler = Base2::StartStopHandlerFactory.get_start_stop_handler(
+            start_stop_handler = CfnManage::StartStopHandlerFactory.get_start_stop_handler(
                 resource['resource_type'],
-                resource['physical_resource_id']
+                resource['physical_resource_id'],
+                @skip_wait
             )
           rescue Exception => e
             $log.error("Error creating start-stop handler for resource of type #{resource['resource_type']}" +