ecs_deploy 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c88001bfd80f75aaeb09c2af4ce4a6e4c44976f465a2812ce1691d4c19e1acd6
-  data.tar.gz: 1bfad93623edc78868040517a1a41006110e8881d661597b6d9b3416449f7b90
+  metadata.gz: 6b008d513950d8b8e5abab3400d9c123dfe22fa7ab0bce2db9fc13d398359ccf
+  data.tar.gz: 8b3096d4cbe25b0febf667b560a0202e915ab9eee2a511b50dc50d61938805b3
 SHA512:
-  metadata.gz: 95f74dd98955021f9280feac7df80a44af75228b0c6460fb807d0b6f8d9976b981a0a253234b36ebd4e5a5f4a10770ba9f77d6782907a9d453e5a4be7bb5c913
-  data.tar.gz: 837b94c1bf33af16303e2cf29720480fa17d4aa165eee2d7b92f9f17fc7facd1637d0475af0b003c5c14ba186042a267a192a46cd9f17dd489909c5839eef83e
+  metadata.gz: 1822d8b4ef6bf0392565918ee6bfe1df2d04e0ad376cb895497fbdd4029523f1db00452dd11f00c3c5acdedafa17039bfd49aebd8b16b4261cffb03a605a5fe9
+  data.tar.gz: 7a395b568354d77868bcd7b9c3d469550f8905fde2640bdace1e014d0639d5ab0a7f6f7b0f94fa6c2928c770a05837783c5994ce702c7fc31c744bc58ee46648

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # v1.0
 
+## Release v1.0.1 - 2021/05/19
+
+### Enhancement
+
+* retry register_task_definition by AWS SDK feature
+  https://github.com/reproio/ecs_deploy/pull/67
+* Support Ruby 3.0
+  https://github.com/reproio/ecs_deploy/pull/66
+* Wait until stop old tasks
+  https://github.com/reproio/ecs_deploy/pull/65
+* Add prioritized_over_upscale_triggers option to triggers
+  https://github.com/reproio/ecs_deploy/pull/62
+* Display only unstable services in EcsDeploy::Service#wait_all_running
+  https://github.com/reproio/ecs_deploy/pull/61
+
 ## Release v1.0.0 - 2019/12/24
 
 ### New feature

data/README.md

@@ -257,6 +257,9 @@ spot_fleet_requests:
         downscale_triggers:
           - alarm_name: "ECS [repro-worker-production] CPUUtilization (low)"
             state: OK
+          - alarm_name: "Aurora DMLLatency is high"
+            state: ALARM
+            prioritized_over_upscale_triggers: true
 
 # If you specify `spot_instance_intrp_warns_queue_urls` as SQS queue for spot instance interruption warnings,
 # autoscaler will polls them and set the state of instances to be intrrupted to "DRAINING".

data/lib/ecs_deploy/auto_scaler/service_config.rb

@@ -45,8 +45,9 @@ module EcsDeploy
           end
         end
 
-        if difference == 0 && desired_count > current_min_task_count
+        if desired_count > current_min_task_count
           downscale_triggers.each do |trigger|
+            next if difference > 0 && !trigger.prioritized_over_upscale_triggers?
             next unless trigger.match?
 
             @logger.info "#{log_prefix} Fire downscale trigger by #{trigger.alarm_name} #{trigger.state}"

data/lib/ecs_deploy/auto_scaler/trigger_config.rb

@@ -5,9 +5,19 @@ require "ecs_deploy/auto_scaler/config_base"
 
 module EcsDeploy
   module AutoScaler
-    TriggerConfig = Struct.new(:alarm_name, :region, :state, :step) do
+    TriggerConfig = Struct.new(:alarm_name, :region, :state, :step, :prioritized_over_upscale_triggers) do
       include ConfigBase
 
+      def match?
+        fetch_alarm.state_value == state
+      end
+
+      def prioritized_over_upscale_triggers?
+        !!prioritized_over_upscale_triggers
+      end
+
+      private
+
       def client
         Aws::CloudWatch::Client.new(
           access_key_id: EcsDeploy.config.access_key_id,
@@ -17,10 +27,6 @@ module EcsDeploy
         )
       end
 
-      def match?
-        fetch_alarm.state_value == state
-      end
-
       def fetch_alarm
         res = client.describe_alarms(alarm_names: [alarm_name])
 

data/lib/ecs_deploy/capistrano.rb

@@ -113,7 +113,7 @@ namespace :ecs do
           service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
           service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
           service_options[:scheduling_strategy] = service[:scheduling_strategy] if service[:scheduling_strategy]
-          s = EcsDeploy::Service.new(service_options)
+          s = EcsDeploy::Service.new(**service_options)
           s.deploy
           s
         end
@@ -178,7 +178,7 @@ namespace :ecs do
           service_options[:deployment_configuration] = service[:deployment_configuration] if service[:deployment_configuration]
           service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
           service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
-          s = EcsDeploy::Service.new(service_options)
+          s = EcsDeploy::Service.new(**service_options)
           s.deploy
           EcsDeploy::TaskDefinition.deregister(current_task_definition_arn, region: r)
           s

data/lib/ecs_deploy/configuration.rb

@@ -13,6 +13,9 @@ module EcsDeploy
     def initialize
       @log_level = :info
       @deploy_wait_timeout = 300
+      # The following values are the default values of Aws::ECS::Waiters::ServicesStable
+      @ecs_wait_until_services_stable_max_attempts = 40
+      @ecs_wait_until_services_stable_delay = 15
     end
   end
 end

data/lib/ecs_deploy/instance_fluctuation_manager.rb

@@ -8,6 +8,7 @@ module EcsDeploy
 
     MAX_UPDATABLE_ECS_CONTAINER_COUNT = 10
     MAX_DETACHEABLE_EC2_INSTACE_COUNT = 20
+    MAX_DESCRIBABLE_ECS_TASK_COUNT = 100
 
     def initialize(region:, cluster:, auto_scaling_group_name:, desired_capacity:, logger:)
       @region = region
@@ -18,7 +19,7 @@ module EcsDeploy
     end
 
     def increase
-      asg = as_client.describe_auto_scaling_groups(auto_scaling_group_names: [@auto_scaling_group_name]).auto_scaling_groups.first
+      asg = fetch_auto_scaling_group
 
       @logger.info("Increase desired capacity of #{@auto_scaling_group_name}: #{asg.desired_capacity} => #{asg.max_size}")
       as_client.update_auto_scaling_group(auto_scaling_group_name: @auto_scaling_group_name, desired_capacity: asg.max_size)
@@ -39,7 +40,7 @@ module EcsDeploy
     end
 
     def decrease
-      asg = as_client.describe_auto_scaling_groups(auto_scaling_group_names: [@auto_scaling_group_name]).auto_scaling_groups.first
+      asg = fetch_auto_scaling_group
 
       decrease_count = asg.desired_capacity - @desired_capacity
       if decrease_count <= 0
@@ -48,13 +49,12 @@ module EcsDeploy
       end
       @logger.info("Decrease desired capacity of #{@auto_scaling_group_name}: #{asg.desired_capacity} => #{@desired_capacity}")
 
-      container_instance_arns = ecs_client.list_container_instances(
-        cluster: @cluster
-      ).container_instance_arns
-      container_instances = ecs_client.describe_container_instances(
-        cluster: @cluster,
-        container_instances: container_instance_arns
-      ).container_instances
+      container_instances = ecs_client.list_container_instances(cluster: @cluster).flat_map do |resp|
+        ecs_client.describe_container_instances(
+          cluster: @cluster,
+          container_instances: resp.container_instance_arns
+        ).container_instances
+      end
 
       az_to_container_instances = container_instances.sort_by {|ci| - ci.running_tasks_count }.group_by do |ci|
         ci.attributes.find {|attribute| attribute.name == "ecs.availability-zone" }.value
@@ -66,24 +66,22 @@ module EcsDeploy
 
       target_container_instances = extract_target_container_instances(decrease_count, az_to_container_instances)
 
-      threads = []
+      @logger.info("running tasks: #{ecs_client.list_tasks(cluster: @cluster).task_arns.size}")
       all_running_task_arns = []
       target_container_instances.map(&:container_instance_arn).each_slice(MAX_UPDATABLE_ECS_CONTAINER_COUNT) do |arns|
+        @logger.info(arns)
         ecs_client.update_container_instances_state(
           cluster: @cluster,
           container_instances: arns,
           status: "DRAINING"
         )
         arns.each do |arn|
-          threads << Thread.new(arn) do |a|
-            all_running_task_arns.concat(stop_tasks(a))
-          end
+          all_running_task_arns.concat(list_running_task_arns(arn))
         end
       end
 
-      threads.each(&:join)
-      ecs_client.wait_until(:tasks_stopped, cluster: @cluster, tasks: all_running_task_arns) unless all_running_task_arns.empty?
-      @logger.info("All running tasks are stopped")
+      stop_tasks_not_belonging_service(all_running_task_arns)
+      wait_until_tasks_stopped(all_running_task_arns)
 
       instance_ids = target_container_instances.map(&:ec2_instance_id)
       terminate_instances(instance_ids)
@@ -115,6 +113,10 @@ module EcsDeploy
       @ecs_client ||= Aws::ECS::Client.new(aws_params)
     end
 
+    def fetch_auto_scaling_group
+      as_client.describe_auto_scaling_groups(auto_scaling_group_names: [@auto_scaling_group_name]).auto_scaling_groups.first
+    end
+
     # Extract container instances to terminate considering AZ balance
     def extract_target_container_instances(decrease_count, az_to_container_instances)
       target_container_instances = []
@@ -132,16 +134,36 @@ module EcsDeploy
       target_container_instances
     end
 
-    def stop_tasks(arn)
-      running_task_arns = ecs_client.list_tasks(cluster: @cluster, container_instance: arn).task_arns
+    # list tasks whose desired_status is "RUNNING" or
+    # whoose desired_status is "STOPPED" but last_status is "RUNNING" on the ECS container
+    def list_running_task_arns(container_instance_arn)
+      running_tasks_arn = ecs_client.list_tasks(cluster: @cluster, container_instance: container_instance_arn).flat_map(&:task_arns)
+      stopped_tasks_arn = ecs_client.list_tasks(cluster: @cluster, container_instance: container_instance_arn, desired_status: "STOPPED").flat_map(&:task_arns)
+      stopped_running_task_arns = stopped_tasks_arn.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).flat_map do |arns|
+        ecs_client.describe_tasks(cluster: @cluster, tasks: arns).tasks.select do |task|
+          task.desired_status == "STOPPED" && task.last_status == "RUNNING"
+        end
+      end.map(&:task_arn)
+      running_tasks_arn + stopped_running_task_arns
+    end
+
+    def wait_until_tasks_stopped(task_arns)
+      @logger.info("All old tasks: #{task_arns.size}")
+      task_arns.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).each do |arns|
+        ecs_client.wait_until(:tasks_stopped, cluster: @cluster, tasks: arns)
+      end
+      @logger.info("All old tasks are stopped")
+    end
+
+    def stop_tasks_not_belonging_service(running_task_arns)
+      @logger.info("Running tasks: #{running_task_arns.size}")
       unless running_task_arns.empty?
-        running_tasks = ecs_client.describe_tasks(cluster: @cluster, tasks: running_task_arns).tasks
-        running_tasks.each do |task|
-          ecs_client.stop_task(cluster: @cluster, task: task.task_arn) if task.group.start_with?("family:")
+        running_task_arns.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).each do |arns|
+          ecs_client.describe_tasks(cluster: @cluster, tasks: arns).tasks.each do |task|
+            ecs_client.stop_task(cluster: @cluster, task: task.task_arn) if task.group.start_with?("family:")
+          end
         end
       end
-      @logger.info("Tasks running on #{arn.split('/').last} will be stopped")
-      running_task_arns
     end
 
     def terminate_instances(instance_ids)

data/lib/ecs_deploy/service.rb

@@ -5,6 +5,8 @@ module EcsDeploy
     CHECK_INTERVAL = 5
     MAX_DESCRIBE_SERVICES = 10
 
+    class TooManyAttemptsError < StandardError; end
+
     attr_reader :cluster, :region, :service_name, :delete
 
     def initialize(
@@ -137,35 +139,27 @@ module EcsDeploy
       end
     end
 
-    def wait_running
-      return if @response.nil?
-
-      service = @response.service
-
-      @client.wait_until(:services_stable, cluster: @cluster, services: [service.service_name]) do |w|
-        w.delay = EcsDeploy.config.ecs_wait_until_services_stable_delay if EcsDeploy.config.ecs_wait_until_services_stable_delay
-        w.max_attempts = EcsDeploy.config.ecs_wait_until_services_stable_max_attempts if EcsDeploy.config.ecs_wait_until_services_stable_max_attempts
-
-        w.before_attempt do
-          EcsDeploy.logger.info "wait service stable [#{service.service_name}]"
-        end
-      end
-    end
-
     def self.wait_all_running(services)
-      services.group_by { |s| [s.cluster, s.region] }.each do |(cl, region), ss|
+      services.group_by { |s| [s.cluster, s.region] }.flat_map do |(cl, region), ss|
         client = Aws::ECS::Client.new(region: region)
-        ss.reject(&:delete).map(&:service_name).each_slice(MAX_DESCRIBE_SERVICES) do |chunked_service_names|
-          client.wait_until(:services_stable, cluster: cl, services: chunked_service_names) do |w|
-            w.delay = EcsDeploy.config.ecs_wait_until_services_stable_delay if EcsDeploy.config.ecs_wait_until_services_stable_delay
-            w.max_attempts = EcsDeploy.config.ecs_wait_until_services_stable_max_attempts if EcsDeploy.config.ecs_wait_until_services_stable_max_attempts
-
-            w.before_attempt do
+        ss.reject(&:delete).map(&:service_name).each_slice(MAX_DESCRIBE_SERVICES).map do |chunked_service_names|
+          Thread.new do
+            EcsDeploy.config.ecs_wait_until_services_stable_max_attempts.times do
               EcsDeploy.logger.info "wait service stable [#{chunked_service_names.join(", ")}]"
+              resp = client.describe_services(cluster: cl, services: chunked_service_names)
+              resp.services.each do |s|
+                # cf. https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-ecs/lib/aws-sdk-ecs/waiters.rb#L91-L96
+                if s.deployments.size == 1 && s.running_count == s.desired_count
+                  chunked_service_names.delete(s.service_name)
+                end
+              end
+              break if chunked_service_names.empty?
+              sleep EcsDeploy.config.ecs_wait_until_services_stable_delay
             end
+            raise TooManyAttemptsError unless chunked_service_names.empty?
           end
         end
-      end
+      end.each(&:join)
     end
 
     private

data/lib/ecs_deploy/task_definition.rb

@@ -1,8 +1,15 @@
 module EcsDeploy
   class TaskDefinition
+    RETRY_BACKOFF = lambda do |c|
+      sleep(1)
+    end
+
+    RETRY_LIMIT = 10
+
     def self.deregister(arn, region: nil)
       region ||= EcsDeploy.config.default_region
-      client = region ? Aws::ECS::Client.new(region: region) : Aws::ECS::Client.new
+      param = {retry_backoff: RETRY_BACKOFF, retry_limit: RETRY_LIMIT}
+      client = region ? Aws::ECS::Client.new(param.merge(region: region)) : Aws::ECS::Client.new(param)
       client.deregister_task_definition({
         task_definition: arn,
       })
@@ -39,8 +46,8 @@ module EcsDeploy
       @cpu = cpu&.to_s
       @memory = memory&.to_s
       @tags = tags
-
-      @client = region ? Aws::ECS::Client.new(region: region) : Aws::ECS::Client.new
+      param = {retry_backoff: RETRY_BACKOFF, retry_limit: RETRY_LIMIT}
+      @client = region ? Aws::ECS::Client.new(param.merge(region: region)) : Aws::ECS::Client.new(param)
       @region = @client.config.region
     end
 

data/lib/ecs_deploy/version.rb

@@ -1,3 +1,3 @@
 module EcsDeploy
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ecs_deploy
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - joker1007
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2021-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-autoscaling
@@ -207,7 +207,7 @@ files:
 homepage: https://github.com/reproio/ecs_deploy
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -222,8 +222,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.0
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: AWS ECS deploy helper
 test_files: []