ecs_deploy 0.3.2 → 1.0.0

data/lib/ecs_deploy/auto_scaler/service_config.rb

@@ -0,0 +1,222 @@
+require "aws-sdk-ecs"
+require "ecs_deploy"
+require "ecs_deploy/auto_scaler/config_base"
+require "ecs_deploy/auto_scaler/trigger_config"
+
+module EcsDeploy
+  module AutoScaler
+    SERVICE_CONFIG_ATTRIBUTES = %i(name cluster region step max_task_count min_task_count idle_time scheduled_min_task_count cooldown_time_for_reach_max upscale_triggers downscale_triggers desired_count required_capacity)
+    ServiceConfig = Struct.new(*SERVICE_CONFIG_ATTRIBUTES) do
+      include ConfigBase
+
+      MAX_DESCRIBABLE_TASK_COUNT = 100
+
+      def initialize(attributes = {}, logger)
+        super
+        self.idle_time ||= 60
+        self.max_task_count = Array(max_task_count)
+        self.upscale_triggers = upscale_triggers.to_a.map do |t|
+          TriggerConfig.new({"region" => region, "step" => step}.merge(t), logger)
+        end
+        self.downscale_triggers = downscale_triggers.to_a.map do |t|
+          TriggerConfig.new({"region" => region, "step" => step}.merge(t), logger)
+        end
+        self.max_task_count.sort!
+        self.desired_count = fetch_service.desired_count
+        self.required_capacity ||= 1
+        @reach_max_at = nil
+        @last_updated_at = nil
+        @logger = logger
+      end
+
+      def adjust_desired_count(cluster_resource_manager)
+        if idle?
+          @logger.debug "#{name} is idling"
+          return
+        end
+
+        difference = 0
+        upscale_triggers.each do |trigger|
+          next if difference >= trigger.step
+
+          if trigger.match?
+            @logger.info "#{log_prefix} Fire upscale trigger by #{trigger.alarm_name} #{trigger.state}"
+            difference = trigger.step
+          end
+        end
+
+        if difference == 0 && desired_count > current_min_task_count
+          downscale_triggers.each do |trigger|
+            next unless trigger.match?
+
+            @logger.info "#{log_prefix} Fire downscale trigger by #{trigger.alarm_name} #{trigger.state}"
+            difference = [difference, -trigger.step].min
+          end
+        end
+
+        if current_min_task_count > desired_count + difference
+          difference = current_min_task_count - desired_count
+        end
+
+        if difference >= 0 && desired_count > max_task_count.max
+          difference = max_task_count.max - desired_count
+        end
+
+        if difference != 0
+          update_service(difference, cluster_resource_manager)
+        end
+      end
+
+      def wait_until_desired_count_updated
+        @increase_desired_count_thread&.join
+      rescue => e
+        AutoScaler.error_logger.warn("`#{__method__}': #{e} (#{e.class})")
+      ensure
+        @increase_desired_count_thread = nil
+      end
+
+      private
+
+      def client
+        Aws::ECS::Client.new(
+          access_key_id: EcsDeploy.config.access_key_id,
+          secret_access_key: EcsDeploy.config.secret_access_key,
+          region: region,
+          logger: logger
+        )
+      end
+
+      def idle?
+        return false unless @last_updated_at
+
+        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - @last_updated_at
+        diff < idle_time
+      end
+
+      def current_min_task_count
+        return min_task_count if scheduled_min_task_count.nil? || scheduled_min_task_count.empty?
+
+        scheduled_min_task_count.find(-> { {"count" => min_task_count} }) { |s|
+          from = Time.parse(s["from"])
+          to = Time.parse(s["to"])
+          (from..to).cover?(Time.now)
+        }["count"]
+      end
+
+      def overheat?
+        return false unless @reach_max_at
+        (Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - @reach_max_at) > cooldown_time_for_reach_max
+      end
+
+      def fetch_service
+        res = client.describe_services(cluster: cluster, services: [name])
+        raise "Service \"#{name}\" is not found" if res.services.empty?
+        res.services[0]
+      rescue => e
+        AutoScaler.error_logger.error(e)
+      end
+
+      def update_service(difference, cluster_resource_manager)
+        next_desired_count = desired_count + difference
+        current_level = max_task_level(desired_count)
+        next_level = max_task_level(next_desired_count)
+        if current_level < next_level && overheat? # next max
+          level = next_level
+          @reach_max_at = nil
+          @logger.info "#{log_prefix} Service is overheat, uses next max count"
+        elsif current_level < next_level && !overheat? # wait cooldown
+          level = current_level
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          @reach_max_at ||= now
+          @logger.info "#{log_prefix} Service waits cooldown elapsed #{(now - @reach_max_at).to_i}sec"
+        elsif current_level == next_level && next_desired_count >= max_task_count[current_level] # reach current max
+          level = current_level
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          @reach_max_at ||= now
+          @logger.info "#{log_prefix} Service waits cooldown elapsed #{(now - @reach_max_at).to_i}sec"
+        elsif current_level == next_level && next_desired_count < max_task_count[current_level]
+          level = current_level
+          @reach_max_at = nil
+          @logger.info "#{log_prefix} Service clears cooldown state"
+        elsif current_level > next_level
+          level = next_level
+          @reach_max_at = nil
+          @logger.info "#{log_prefix} Service clears cooldown state"
+        end
+
+        next_desired_count = [next_desired_count, max_task_count[level]].min
+        if next_desired_count > desired_count
+          increase_desired_count(next_desired_count - desired_count, cluster_resource_manager)
+        else
+          decrease_desired_count(desired_count - next_desired_count, cluster_resource_manager)
+        end
+
+        @last_updated_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        @logger.info "#{log_prefix} Update desired_count to #{next_desired_count}"
+      rescue => e
+        AutoScaler.error_logger.error(e)
+      end
+
+      def increase_desired_count(by, cluster_resource_manager)
+        applied_desired_count = desired_count
+        self.desired_count += by
+
+        wait_until = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 180
+        @increase_desired_count_thread = Thread.new do
+          cl = client
+          by.times do
+            timeout = wait_until - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            break if timeout <= 0
+            break unless cluster_resource_manager.acquire(required_capacity, timeout: timeout)
+            begin
+              cl.update_service(cluster: cluster, service: name, desired_count: applied_desired_count + 1)
+              applied_desired_count += 1
+            rescue => e
+              cluster_resource_manager.release(required_capacity)
+              AutoScaler.error_logger.error(e)
+              break
+            end
+          end
+
+          if applied_desired_count != desired_count
+            self.desired_count = applied_desired_count
+            @logger.info "#{log_prefix} Failed to update service and set desired_count to #{desired_count}"
+          end
+        end
+      end
+
+      def decrease_desired_count(by, cluster_resource_manager)
+        cl = client
+        running_task_arns = cl.list_tasks(cluster: cluster, service_name: name, desired_status: "RUNNING").flat_map(&:task_arns)
+
+        cl.update_service(cluster: cluster, service: name, desired_count: desired_count - by)
+
+        cl.wait_until(:services_stable, cluster: cluster, services: [name]) do |w|
+          w.before_wait do
+            @logger.debug "#{log_prefix} wait service stable"
+          end
+        end
+
+        stopping_task_arns = running_task_arns - cl.list_tasks(cluster: cluster, service_name: name, desired_status: "RUNNING").flat_map(&:task_arns)
+        stopping_task_arns.each_slice(MAX_DESCRIBABLE_TASK_COUNT) do |arns|
+          cl.wait_until(:tasks_stopped, cluster: cluster, tasks: arns) do |w|
+            w.before_wait do
+              @logger.debug "#{log_prefix} wait stopping tasks stopped"
+            end
+          end
+        end
+
+        cluster_resource_manager.release(required_capacity * by)
+        self.desired_count -= by
+      end
+
+      def max_task_level(count)
+        max_task_count.index { |i| count <= i } || max_task_count.size - 1
+      end
+
+      def log_prefix
+        "[#{self.class.to_s.sub(/\AEcsDeploy::AutoScaler::/, "")} #{name} #{region}]"
+      end
+    end
+  end
+end

data/lib/ecs_deploy/auto_scaler/spot_fleet_request_config.rb

@@ -0,0 +1,102 @@
+require "json"
+require "timeout"
+
+require "aws-sdk-ec2"
+require "ecs_deploy"
+require "ecs_deploy/auto_scaler/config_base"
+require "ecs_deploy/auto_scaler/cluster_resource_manager"
+
+module EcsDeploy
+  module AutoScaler
+    SpotFleetRequestConfig = Struct.new(:id, :region, :cluster, :buffer, :service_configs) do
+      include ConfigBase
+
+      def initialize(attributes = {}, logger)
+        attributes = attributes.dup
+        services = attributes.delete("services")
+        super(attributes, logger)
+        self.service_configs = services.map do |s|
+          ServiceConfig.new(s.merge("cluster" => cluster, "region" => region), logger)
+        end
+      end
+
+      def name
+        id
+      end
+
+      def update_desired_capacity(required_capacity)
+        terminate_orphan_instances
+
+        desired_capacity = (required_capacity + buffer.to_f).ceil
+
+        request_config = ec2_client.describe_spot_fleet_requests(
+          spot_fleet_request_ids: [id]
+        ).spot_fleet_request_configs[0].spot_fleet_request_config
+
+        return if desired_capacity == request_config.target_capacity
+
+        ec2_client.modify_spot_fleet_request(spot_fleet_request_id: id, target_capacity: desired_capacity)
+
+        cluster_resource_manager.trigger_capacity_update(
+          request_config.target_capacity,
+          desired_capacity,
+          # Wait until the capacity is updated to prevent the process from terminating before container draining is completed
+          wait_until_capacity_updated: desired_capacity < request_config.target_capacity,
+        )
+        @logger.info "#{log_prefix} Update desired_capacity to #{desired_capacity}"
+      rescue => e
+        AutoScaler.error_logger.error(e)
+      end
+
+      def cluster_resource_manager
+        @cluster_resource_manager ||= EcsDeploy::AutoScaler::ClusterResourceManager.new(
+          region: region,
+          cluster: cluster,
+          service_configs: service_configs,
+          capacity_based_on: "vCPUs",
+          logger: @logger,
+        )
+      end
+
+      private
+
+      def terminate_orphan_instances
+        container_instance_ids = cluster_resource_manager.fetch_container_instances_in_cluster.map(&:ec2_instance_id)
+        spot_fleet_instances = ec2_client.describe_spot_fleet_instances(spot_fleet_request_id: id).active_instances
+        orphans = spot_fleet_instances.reject { |i| container_instance_ids.include?(i.instance_id) }.map(&:instance_id)
+
+        return if orphans.empty?
+
+        running_instances = ec2_client.describe_instances(
+          instance_ids: orphans,
+          filters: [{ name: "instance-state-name", values: ["running"] }],
+        ).reservations.flat_map(&:instances)
+        # instances which have just launched might not be registered to the cluster yet.
+        instance_ids = running_instances.select { |i| (Time.now - i.launch_time) > 600 }.map(&:instance_id)
+
+        return if instance_ids.empty?
+
+        # Terminate orpahns without canceling spot instance request
+        # because we can't terminate canceled spot instances by decreasing the capacity
+        ec2_client.terminate_instances(instance_ids: instance_ids)
+
+        @logger.info "#{log_prefix} Terminated instances: #{instance_ids.inspect}"
+      rescue => e
+        AutoScaler.error_logger.error(e)
+      end
+
+      def ec2_client
+        Aws::EC2::Client.new(
+          access_key_id: EcsDeploy.config.access_key_id,
+          secret_access_key: EcsDeploy.config.secret_access_key,
+          region: region,
+          logger: logger,
+        )
+      end
+
+      def log_prefix
+        "[#{self.class.to_s.sub(/\AEcsDeploy::AutoScaler::/, "")} #{name} #{region}]"
+      end
+    end
+  end
+end

data/lib/ecs_deploy/auto_scaler/trigger_config.rb

@@ -0,0 +1,36 @@
+require "aws-sdk-cloudwatch"
+require "ecs_deploy"
+require "ecs_deploy/auto_scaler"
+require "ecs_deploy/auto_scaler/config_base"
+
+module EcsDeploy
+  module AutoScaler
+    TriggerConfig = Struct.new(:alarm_name, :region, :state, :step) do
+      include ConfigBase
+
+      def client
+        Aws::CloudWatch::Client.new(
+          access_key_id: EcsDeploy.config.access_key_id,
+          secret_access_key: EcsDeploy.config.secret_access_key,
+          region: region,
+          logger: logger
+        )
+      end
+
+      def match?
+        fetch_alarm.state_value == state
+      end
+
+      def fetch_alarm
+        res = client.describe_alarms(alarm_names: [alarm_name])
+
+        raise "Alarm \"#{alarm_name}\" is not found" if res.metric_alarms.empty?
+        res.metric_alarms[0].tap do |alarm|
+          AutoScaler.logger.debug("#{alarm.alarm_name} state is #{alarm.state_value}")
+        end
+      rescue => e
+        AutoScaler.error_logger.error(e)
+      end
+    end
+  end
+end

data/lib/ecs_deploy/capistrano.rb

@@ -1,4 +1,5 @@
 require 'ecs_deploy'
+require 'ecs_deploy/instance_fluctuation_manager'
 
 namespace :ecs do
   task :configure do
@@ -7,6 +8,8 @@ namespace :ecs do
       c.deploy_wait_timeout = fetch(:ecs_deploy_wait_timeout) if fetch(:ecs_deploy_wait_timeout)
       c.ecs_service_role = fetch(:ecs_service_role) if fetch(:ecs_service_role)
       c.default_region = Array(fetch(:ecs_region))[0] if fetch(:ecs_region)
+      c.ecs_wait_until_services_stable_max_attempts = fetch(:ecs_wait_until_services_stable_max_attempts) if fetch(:ecs_wait_until_services_stable_max_attempts)
+      c.ecs_wait_until_services_stable_delay = fetch(:ecs_wait_until_services_stable_delay) if fetch(:ecs_wait_until_services_stable_delay)
     end
 
     if ENV["TARGET_CLUSTER"]
@@ -20,7 +23,7 @@ namespace :ecs do
   task register_task_definition: [:configure] do
     if fetch(:ecs_tasks)
       regions = Array(fetch(:ecs_region))
-      regions = [EcsDeploy.config.default_region || ENV["AWS_DEFAULT_REGION"]] if regions.empty?
+      regions = [EcsDeploy.config.default_region] if regions.empty?
       ecs_registered_tasks = {}
       regions.each do |region|
         ecs_registered_tasks[region] = {}
@@ -30,9 +33,14 @@ namespace :ecs do
             task_definition_name: t[:name],
             container_definitions: t[:container_definitions],
             task_role_arn: t[:task_role_arn],
+            execution_role_arn: t[:execution_role_arn],
             volumes: t[:volumes],
             network_mode: t[:network_mode],
             placement_constraints: t[:placement_constraints],
+            requires_compatibilities: t[:requires_compatibilities],
+            cpu: t[:cpu],
+            memory: t[:memory],
+            tags: t[:tags],
           )
           result = task_definition.register
           ecs_registered_tasks[region][t[:name]] = result
@@ -58,6 +66,10 @@ namespace :ecs do
             description: t[:description],
             target_id: t[:target_id],
             task_definition_name: t[:task_definition_name],
+            network_configuration: t[:network_configuration],
+            launch_type: t[:launch_type],
+            platform_version: t[:platform_version],
+            group: t[:group],
             revision: t[:revision],
             task_count: t[:task_count],
             role_arn: t[:role_arn],
@@ -89,8 +101,18 @@ namespace :ecs do
             task_definition_name: service[:task_definition_name],
             load_balancers: service[:load_balancers],
             desired_count: service[:desired_count],
+            launch_type: service[:launch_type],
+            network_configuration: service[:network_configuration],
+            health_check_grace_period_seconds: service[:health_check_grace_period_seconds],
+            delete: service[:delete],
+            enable_ecs_managed_tags: service[:enable_ecs_managed_tags],
+            tags: service[:tags],
+            propagate_tags: service[:propagate_tags],
           }
           service_options[:deployment_configuration] = service[:deployment_configuration] if service[:deployment_configuration]
+          service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
+          service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
+          service_options[:scheduling_strategy] = service[:scheduling_strategy] if service[:scheduling_strategy]
           s = EcsDeploy::Service.new(service_options)
           s.deploy
           s
@@ -149,8 +171,13 @@ namespace :ecs do
             task_definition_name: rollback_arn,
             load_balancers: service[:load_balancers],
             desired_count: service[:desired_count],
+            launch_type: service[:launch_type],
+            network_configuration: service[:network_configuration],
+            health_check_grace_period_seconds: service[:health_check_grace_period_seconds],
           }
           service_options[:deployment_configuration] = service[:deployment_configuration] if service[:deployment_configuration]
+          service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
+          service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
           s = EcsDeploy::Service.new(service_options)
           s.deploy
           EcsDeploy::TaskDefinition.deregister(current_task_definition_arn, region: r)
@@ -160,4 +187,46 @@ namespace :ecs do
       end
     end
   end
+
+  task increase_instances_to_max_size: [:configure] do
+    configs = fetch(:ecs_instance_fluctuation_manager_configs, [])
+    unless configs.empty?
+      regions = Array(fetch(:ecs_region))
+      regions = [EcsDeploy.config.default_region] if regions.empty?
+      regions.each do |region|
+        configs.each do |config|
+          logger = config.fetch(:logger, EcsDeploy.logger)
+          m = EcsDeploy::InstanceFluctuationManager.new(
+            region: config[:region] || region,
+            cluster: config[:cluster] || fetch(:ecs_default_cluster),
+            auto_scaling_group_name: config[:auto_scaling_group_name],
+            desired_capacity: config[:desired_capacity],
+            logger: logger
+          )
+          m.increase
+        end
+      end
+    end
+  end
+
+  task terminate_redundant_instances: [:configure] do
+    configs = fetch(:ecs_instance_fluctuation_manager_configs, [])
+    unless configs.empty?
+      regions = Array(fetch(:ecs_region))
+      regions = [EcsDeploy.config.default_region] if regions.empty?
+      regions.each do |region|
+        configs.each do |config|
+          logger = config.fetch(:logger, EcsDeploy.logger)
+          m = EcsDeploy::InstanceFluctuationManager.new(
+            region: config[:region] || region,
+            cluster: config[:cluster] || fetch(:ecs_default_cluster),
+            auto_scaling_group_name: config[:auto_scaling_group_name],
+            desired_capacity: config[:desired_capacity],
+            logger: logger
+          )
+          m.decrease
+        end
+      end
+    end
+  end
 end