cfn_manage 0.7.0 → 0.8.3

data/lib/cfn_manage/handlers/documentdb.rb

@@ -0,0 +1,89 @@
+require 'aws-sdk-docdb'
+require 'cfn_manage/aws_credentials'
+
+module CfnManage
+  module StartStopHandler
+    class DocumentDb
+
+      def initialize(cluster_id, options = {})
+        @cluster_id = cluster_id
+        credentials = CfnManage::AWSCredentials.get_session_credentials("startstopcluster_#{cluster_id}")
+        @docdb_client = Aws::DocDB::Client.new(retry_limit: 20)
+        if credentials != nil
+          @docdb_client = Aws::DocDB::Client.new(credentials: credentials, retry_limit: 20)
+        end
+        cluster = @docdb_client.describe_db_clusters({ db_cluster_identifier: @cluster_id })
+        @docdb_cluster = cluster.db_clusters.first
+      end
+
+      def start(configuration)
+        if @docdb_cluster.status == 'available'
+          $log.info("DocDB Cluster #{@cluster_id} is already in available state")
+          return
+        end
+
+        # start docdb cluster
+        if @docdb_cluster.status == 'stopped'
+          $log.info("Starting DocDB cluster #{@cluster_id}")
+          @docdb_client.start_db_cluster({ db_cluster_identifier: @cluster_id })
+          unless CfnManage.skip_wait?
+            # wait cluster to become available
+            $log.info("Waiting DocDB cluster to become available #{@cluster_id}")
+            wait('available')
+          end
+        else
+          $log.info("DocDB Cluster #{@cluster_id} is not in a stopped state. State: #{@docdb_cluster.status}")
+        end
+      end
+
+      def stop
+        if @docdb_cluster.status == 'stopped'
+          $log.info("DocDB Cluster #{@cluster_id} is already stopped")
+          return {}
+        end
+
+        if @docdb_cluster.status != 'available'
+          $log.info("DocDB Cluster #{@cluster_id} is not in a available state. State: #{@docdb_cluster.status}")
+          return {}
+        end
+        # stop docdb cluster and wait for it to be fully stopped
+        $log.info("Stopping DocDB cluster #{@cluster_id}")
+        @docdb_client.stop_db_cluster({ db_cluster_identifier: @cluster_id })
+        unless CfnManage.skip_wait?
+          $log.info("Waiting DocDB cluster to be stopped #{@cluster_id}")
+          wait('stopped')
+        end
+        return {}
+      end
+
+      def wait(completed_state)
+        # reached state must be steady, at least a minute.
+        state_count = 0
+        steady_count = 4
+        attempts = 0
+
+        until attempts == (max_attempts = 60*6) do
+          # Declare client and cluster variable a second time inside the loop so it re-evaluates each time.
+          docdb = @docdb_client.describe_db_clusters({ db_cluster_identifier: @cluster_id })
+          cluster = docdb.db_clusters.first
+          $log.info("DocDB Cluster #{cluster.db_cluster_identifier} state: #{cluster.status}, waiting for #{completed_state}")
+
+          if cluster.status == "#{completed_state}"
+            state_count = state_count + 1
+            $log.info("#{state_count}/#{steady_count}")
+          else
+            state_count = 0
+          end
+          break if state_count == steady_count
+          attempts = attempts + 1
+          sleep(15)
+        end
+
+        if attempts == max_attempts
+          $log.error("DocDB Cluster #{@cluster_id} did not enter #{completed_state} state, however continuing operations...")
+        end
+      end
+
+    end
+  end
+end

data/lib/cfn_manage/handlers/ec2.rb

@@ -0,0 +1,42 @@
+require 'aws-sdk-ec2'
+require 'cfn_manage/aws_credentials'
+
+module CfnManage
+  module StartStopHandler
+    class Ec2
+
+      def initialize(instance_id, options = {})
+        credentials = CfnManage::AWSCredentials.get_session_credentials("stoprun_#{instance_id}")
+        ec2_client = Aws::EC2::Client.new(credentials: credentials, retry_limit: 20)
+        @instance = Aws::EC2::Resource.new(client: ec2_client, retry_limit: 20).instance(instance_id)
+        @instance_id = instance_id
+      end
+
+      def start(configuration)
+        if %w(running).include?(@instance.state.name)
+          $log.info("Instance #{@instance_id} already running")
+          return
+        end
+        $log.info("Starting instance #{@instance_id}")
+        @instance.start()
+      end
+
+      def stop
+        if %w(stopped stopping).include?(@instance.state.name)
+          $log.info("Instance #{@instance_id} already stopping or stopped")
+          return
+        end
+        $log.info("Stopping instance #{@instance_id}")
+        @instance.stop()
+
+        # empty configuration for ec2 instances
+        return {}
+      end
+
+      def wait(wait_states=[])
+        $log.debug("Not waiting for EC2 instance #{@instance_id}")
+      end
+
+    end
+  end
+end

data/lib/cfn_manage/handlers/ecs_cluster.rb

@@ -0,0 +1,219 @@
+require 'aws-sdk-ecs'
+require 'cfn_manage/aws_credentials'
+
+module CfnManage
+  module StartStopHandler
+    class EcsCluster
+
+      def initialize(cluster_id, options = {})
+        @wait_state = options.has_key?(:wait_state) ? options[:wait_state] : CfnManage.ecs_wait_state
+        @skip_wait = options.has_key?(:skip_wait) ? CfnManage.true?(options[:skip_wait]) : CfnManage.skip_wait? 
+        @wait_container_instances = options.has_key?(:wait_container_instances) ? CfnManage.true?(options[:wait_container_instances]) : CfnManage.ecs_wait_container_instances? 
+        @ignore_missing_ecs_config = options.has_key?(:ignore_missing_ecs_config) ? CfnManage.true?(options[:ignore_missing_ecs_config]) : CfnManage.ignore_missing_ecs_config?
+        
+        credentials = CfnManage::AWSCredentials.get_session_credentials("stoprun_#{cluster_id}")
+        @ecs_client = Aws::ECS::Client.new(credentials: credentials, retry_limit: 20)
+        @elb_client = Aws::ElasticLoadBalancingV2::Client.new(credentials: credentials, retry_limit: 20)
+        @services = []
+        @ecs_client.list_services(cluster: cluster_id, scheduling_strategy: 'REPLICA', max_results: 100).each do |results|
+          @services.push(*results.service_arns)
+        end
+        $log.info("Found #{@services.count} services in ECS cluster #{cluster_id}")
+        @cluster = cluster_id
+      end
+
+      def start(configuration)
+        if @wait_container_instances
+          wait_for_instances()
+        end
+        
+        @services.each do |service_arn|
+
+          $log.info("Searching for ECS service #{service_arn} in cluster #{@cluster}")
+          service = @ecs_client.describe_services(services:[service_arn], cluster: @cluster).services.first
+
+          if service.desired_count != 0
+            $log.info("ECS service #{service.service_name} is already running")
+            next
+          end
+
+          if configuration.has_key?(service.service_name)
+            desired_count = configuration[service.service_name].has_key?('desired_count') ? configuration[service.service_name]['desired_count'] : 0
+            if desired_count == 0
+              $log.info("Desired count is #{desired_count}, skipping starting of service #{service.service_name}")
+              next
+            end
+          elsif CfnManage.ignore_missing_ecs_config?
+            $log.info("ECS service #{service.service_name} wasn't previously stopped by cfn_manage. Option --ignore-missing-ecs-config set and setting desired count to 1")
+            desired_count = 1
+          else
+            $log.warn("ECS service #{service.service_name} wasn't previously stopped by cfn_manage. Skipping ...")
+            next
+          end
+
+          $log.info("Starting ECS service #{service.service_name} with desired count of #{desired_count}")
+          @ecs_client.update_service({
+            desired_count: desired_count,
+            service: service_arn,
+            cluster: @cluster
+          })
+
+        end
+
+        if !@skip_wait
+          @services.each do |service_arn|
+            service = @ecs_client.describe_services(services:[service_arn], cluster: @cluster).services.first
+
+            if service.desired_count == 0
+              $log.info("ECS service #{service.service_name} was not started so will not wait for it")
+              next
+            end
+            
+            wait(@wait_state,service_arn)
+          end
+        end
+        
+      end
+
+      def stop
+        configuration = {}
+        @services.each do |service_arn|
+
+          $log.info("Searching for ECS service #{service_arn} in cluster #{@cluster}")
+          service = @ecs_client.describe_services(services:[service_arn], cluster: @cluster).services.first
+
+          if service.desired_count == 0
+            $log.info("ECS service #{service.service_name} is already stopped")
+            next
+          end
+
+          configuration[service.service_name] = { desired_count: service.desired_count }
+          $log.info("Stopping ECS service #{service.service_name}")
+          @ecs_client.update_service({
+            desired_count: 0,
+            service: service_arn,
+            cluster: @cluster
+          })
+
+        end
+
+        return configuration.empty? ? nil : configuration
+      end
+
+      def wait(type,service_arn=nil)
+        
+        if service_arn.nil?
+          $log.warn("Unable to wait for #{service_arn} service")
+          return
+        end
+        
+        attempts = 0
+        
+        until attempts == (max_attempts = 60*6) do
+          
+          case type
+          when 'Running'
+            success = wait_till_running(service_arn)
+          when 'HealthyInTargetGroup'
+            success = wait_till_healthy_in_target_group(service_arn)
+          when 'Skip'
+            $log.info("Skipping wait for #{service_arn} service")
+            break
+          else
+            $log.warn("Unknown ECS service wait type #{type}. Skipping...")
+            break
+          end
+          
+          if success
+            break
+          end
+          
+          attempts = attempts + 1
+          sleep(15)
+        end
+
+        if attempts == max_attempts
+          $log.error("Failed to wait for ecs service with wait type #{type}")
+        end
+      end
+      
+      def wait_for_instances
+        
+        attempts = 0
+        
+        until attempts == (max_attempts = 60*3) do
+          
+          resp = @ecs_client.list_container_instances({
+            cluster: @cluster,
+            status: "ACTIVE"
+          })
+          
+          if resp.container_instance_arns.any?
+            $log.info("A container instances has joined ecs cluster #{@cluster}")
+            break
+          end
+          
+          attempts = attempts + 1
+          sleep(5)
+        end
+
+        if attempts == max_attempts
+          $log.error("Failed to wait for container instances to join ecs cluster #{@cluster}")
+        end
+      end
+      
+      def wait_till_running(service_arn)
+        service_name = service_arn.split('/').last
+        service = @ecs_client.describe_services(services:[service_arn], cluster: @cluster).services.first
+                
+        if service.running_count > 0
+          $log.info("ecs service #{service_name} has #{service.running_count} running tasks")
+          return true
+        end  
+        
+        $log.info("waiting for ecs service #{service_name} to reach a running state")
+        return false
+      end
+      
+      def wait_till_healthy_in_target_group(service_arn)
+        service = @ecs_client.describe_services(services:[service_arn], cluster: @cluster).services.first
+        target_groups = service.load_balancers.collect { |lb| lb.target_group_arn }
+        
+        if target_groups.empty?
+          # we want to skip here if the asg is not associated with any target groups
+          $log.info("ecs aervice #{service_arn} is not associated with any target groups")
+          return true
+        end
+        
+        target_health = []
+        target_groups.each do |tg| 
+          resp = @elb_client.describe_target_health({
+            target_group_arn: tg, 
+          })
+          if resp.target_health_descriptions.empty?
+            # we need to wait until a ecs task has been placed into the target group
+            # before we can check it's healthy
+            $log.info("ECS service #{service_arn} hasn't been placed into target group #{tg.split('/')[1]} yet")
+            return false
+          end
+          target_health.push(*resp.target_health_descriptions)
+        end
+              
+        state = target_health.collect {|tg| tg.target_health.state}
+                
+        if state.all? 'healthy'
+          $log.info("All ecs tasks are in a healthy state in target groups #{target_groups.map {|tg| tg.split('/')[1] }}")
+          return true
+        end
+        
+        unhealthy = target_health.select {|tg| tg.target_health.state != 'healthy'}
+        unhealthy.each do |tg|
+          $log.info("waiting for ecs task #{tg.target.id} to be healthy in target group. Current state is #{tg.target_health.state}")
+        end
+        
+        return false
+      end
+
+    end
+  end
+end

data/lib/cfn_manage/handlers/rds.rb

@@ -0,0 +1,142 @@
+require 'aws-sdk-rds'
+require 'cfn_manage/aws_credentials'
+
+module CfnManage
+  module StartStopHandler
+    class Rds
+
+      def initialize(instance_id, options = {})
+        @instance_id = instance_id
+        @excluded_engines = %w(aurora aurora-mysql aurora-postgresql) # RDS list of exluded engines that don't support RDS stop start
+        credentials = CfnManage::AWSCredentials.get_session_credentials("startstoprds_#{instance_id}")
+        @rds_client = Aws::RDS::Client.new(retry_limit: 20)
+        if credentials != nil
+          @rds_client = Aws::RDS::Client.new(credentials: credentials, retry_limit: 20)
+        end
+        rds = Aws::RDS::Resource.new(client: @rds_client)
+        @rds_instance = rds.db_instance(instance_id)
+
+      end
+
+      def start(configuration)
+        if @excluded_engines.include? @rds_instance.engine
+           $log.info("RDS Instance #{@instance_id} engine is #{@rds_instance.engine} and cannot be started by instance.")
+           return
+        end
+
+        if @rds_instance.db_instance_status == 'available'
+          $log.info("RDS Instance #{@instance_id} is already in available state")
+        end
+
+        # start rds instance
+        if @rds_instance.db_instance_status == 'stopped'
+          $log.info("Starting db instance #{@instance_id}")
+          @rds_client.start_db_instance({ db_instance_identifier: @instance_id })
+
+          # wait instance to become available
+          unless CfnManage.skip_wait?
+            $log.info("Waiting db instance to become available #{@instance_id}")
+            wait('available')
+          end
+        else
+          wait('available') unless CfnManage.skip_wait?
+        end
+
+        # convert rds instance to mutli-az if required
+        if configuration['is_multi_az']
+          $log.info("Converting to Multi-AZ instance after start (instance #{@instance_id})")
+          set_rds_instance_multi_az( true)
+        end unless configuration.nil?
+      end
+
+      def stop
+
+        configuration = {
+            is_multi_az: @rds_instance.multi_az
+        }
+        # RDS list of exluded engines that don't support RDS stop start
+        if @excluded_engines.include? @rds_instance.engine
+           $log.info("RDS Instance #{@instance_id} engine is #{@rds_instance.engine} and cannot be stoped by instance.")
+           return configuration
+        end
+
+        # check if available
+        if @rds_instance.db_instance_status != 'available'
+          $log.warn("RDS Instance #{@instance_id} not in available state, and thus can not be stopped")
+          $log.warn("RDS Instance #{@instance_id} state: #{@rds_instance.db_instance_status}")
+          return configuration
+        end
+
+        # check if already stopped
+        if @rds_instance.db_instance_status == 'stopped'
+          $log.info("RDS Instance #{@instance_id} is already stopped")
+          return configuration
+        end
+
+        #check if mutli-az RDS. if so, convert to single-az
+        if @rds_instance.multi_az
+          $log.info("Converting to Non-Multi-AZ instance before stop (instance #{@instance_id}")
+          set_rds_instance_multi_az(false)
+        end
+
+        # stop rds instance and wait for it to be fully stopped
+        $log.info("Stopping instance #{@instance_id}")
+        begin
+          @rds_client.stop_db_instance({ db_instance_identifier: @instance_id })
+        rescue Aws::RDS::Errors::InvalidDBInstanceState => e
+          if e.message == "Cannot stop or start a Read-Replica instance"
+            $log.warn("Skipping due to error: #{e.message}") 
+            return
+          else
+            raise e
+          end
+        end
+        unless CfnManage.skip_wait?
+          $log.info("Waiting db instance to be stopped #{@instance_id}")
+          wait('stopped')
+        end
+        return configuration
+      end
+
+      def set_rds_instance_multi_az(multi_az)
+        if @rds_instance.multi_az == multi_az
+          $log.info("Rds instance #{@rds_instance.db_instance_identifier} already multi-az=#{multi_az}")
+          return
+        end
+        @rds_instance.modify({ multi_az: multi_az, apply_immediately: true })
+        # allow half an hour for instance to be converted
+        wait('available')
+      end
+
+      def wait(completed_state)
+        # reached state must be steady, at least a minute. Modifying an instance to/from MultiAZ can't be shorter
+        # than 40 seconds, hence steady count is 4
+        state_count = 0
+        steady_count = 4
+        attempts = 0
+        rds = Aws::RDS::Resource.new(client: @rds_client)
+        until attempts == (max_attempts = 60*6) do
+          instance = rds.db_instance(@instance_id)
+          $log.info("Instance #{instance.db_instance_identifier} state: #{instance.db_instance_status}, waiting for #{completed_state}")
+
+          if instance.db_instance_status == "#{completed_state}"
+            state_count = state_count + 1
+            $log.info("#{state_count}/#{steady_count}")
+          else
+            state_count = 0
+          end
+          break if state_count == steady_count
+          attempts = attempts + 1
+          sleep(15)
+        end
+
+        if attempts == max_attempts
+          $log.error("RDS Database Instance #{@instance_id} did not enter #{state} state, however continuing operations...")
+        end
+      end
+
+      private :set_rds_instance_multi_az
+
+    end
+  end
+end