ecs_deploy 1.0.2 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3988729fd28cd6cfb8fb6f36869855b1bbc468899d453a38e7415552a81c13aa
-  data.tar.gz: 73543aa973eadf609e049355064cd98e6ce18c448b69be6c1920e6cfee9fffa7
+  metadata.gz: a731ec39a6928bb4365fbb4784b91dd92737eb5b5796da19e3be8185d6bc10b8
+  data.tar.gz: '04758ae740d9ab1bb9108d5dcae1df72102fd161e0f96a38f1e30b461415bc7f'
 SHA512:
-  metadata.gz: 95495a3d6fdfaaaf4c7e5197f2f2107905310a0df36dc26197a43e8defe4e1721b9de45eed68a89e8aa50d3187df54590814544806b0e1b719e387bc774005af
-  data.tar.gz: 5e09d4a796fa051db972f5fd70c52ceb96058690123a3cc583aaf4642ebabf9e80cd225afcf64e67af9078944cb4ba013142f9923d9b6d8549d654401fa5edb6
+  metadata.gz: 429dc1f441f7a67a973293adf8c48a62671ab3f5710cee596186a7fe5773b772b643b9ef879fb0b2143d7f7100754b6521ec6823094122fec85bfd189ca3c306
+  data.tar.gz: 03f568218e5e77e2c9b86560ac5dd4f41e19440f98da1ee102ee09057bc7ec8fdf7caa5366b56d06946d5887a0a60d8bae0c1ce04e80e0a50df7a51e40642da0

data/.github/workflows/test.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.5', '2.6', '2.7', '3.0', '3.1', '3.2']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Run tests
+      run: bundle exec rake

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # v1.0
 
+## Release v1.0.4 - 2023/02/10
+
+### Bug fixes
+
+- Fix Aws::AutoScaling::Errors::ValidationError https://github.com/reproio/ecs_deploy/pull/85
+
+- Fix Timeout::Error that occurs in trigger_capacity_update https://github.com/reproio/ecs_deploy/pull/80
+
+- use force a new deployment, when switching from launch type to capacity provider strategy on an existing service https://github.com/reproio/ecs_deploy/pull/75
+
+### Enhancement
+
+- Run test with Ruby 3.2 https://github.com/reproio/ecs_deploy/pull/83
+
+- Merge `propagate_tags` to service_options when updating service https://github.com/reproio/ecs_deploy/pull/82
+
+- Show service event logs while waiting for services https://github.com/reproio/ecs_deploy/pull/81
+
+- Stop supporting ruby 2.4 https://github.com/reproio/ecs_deploy/pull/79
+
+- Display warning that desired count has reached max value https://github.com/reproio/ecs_deploy/pull/78
+
+- Make draining feature opt-outable https://github.com/reproio/ecs_deploy/pull/77
+
+- Add capacity_provider_strategy options to Service https://github.com/reproio/ecs_deploy/pull/74
+
+## Release v1.0.3 - 2021/11/17
+
+### Bug fixes
+* Fix bug that `InstanceFluctuationManager#decrement` tries to detach instances whose status is 'DEREGISTERING'
+  https://github.com/reproio/ecs_deploy/pull/72
+
+### Enhancement
+* Add a cluster name to deployment logs
+  https://github.com/reproio/ecs_deploy/pull/71
+
+
 ## Release v1.0.2 - 2021/05/26
 
 ### Enhancement

data/README.md

@@ -215,6 +215,7 @@ auto_scaling_groups:
     # autoscaler will set the capacity to (buffer + desired_tasks * required_capacity).
     # Adjust this value if it takes much time to prepare ECS instances and launch new tasks.
     buffer: 1
+    disable_draining: false # cf. spot_instance_intrp_warns_queue_urls
     services:
       - name: repro-api-production
         step: 1
@@ -242,6 +243,7 @@ spot_fleet_requests:
     region: ap-northeast-1
     cluster: ecs-cluster-for-worker
     buffer: 1
+    disable_draining: false # cf. spot_instance_intrp_warns_queue_urls
     services:
       - name: repro-worker-production
         step: 1
@@ -261,11 +263,15 @@ spot_fleet_requests:
             state: ALARM
             prioritized_over_upscale_triggers: true
 
-# If you specify `spot_instance_intrp_warns_queue_urls` as SQS queue for spot instance interruption warnings,
-# autoscaler will polls them and set the state of instances to be intrrupted to "DRAINING".
-# autoscaler will also waits for the capacity of active instances in the cluster being decreased
-# when the capacity of spot fleet request is decreased,
-# so you should specify URLs or set the state of the instances to "DRAINING" manually.
+# When you use spot instances, instances that receive interruption warnings should be drained.
+# If you set URLs of SQS queues for spot instance interruption warnings to `spot_instance_intrp_warns_queue_urls`,
+# autoscaler drains instances to interrupt and detaches the instances from the auto scaling groups with
+# should_decrement_desired_capacity false.
+# If you set ECS_ENABLE_SPOT_INSTANCE_DRAINING to true, we recommend that you opt out of the draining feature
+# by setting disable_draining to true in the configurations of auto scaling groups and spot fleet requests.
+# Otherwise, instances don't seem to be drained on rare occasions.
+# Even if you opt out of the feature, you still have the advantage of setting `spot_instance_intrp_warns_queue_urls`
+# because instances to interrupt are replaced with new instances as soon as possible.
 spot_instance_intrp_warns_queue_urls:
   - https://sqs.ap-northeast-1.amazonaws.com/<account-id>/spot-instance-intrp-warns
 ```

data/ecs_deploy.gemspec

@@ -30,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", ">= 1.11", "< 3"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rexml" # For aws-sdk-*
 end

data/lib/ecs_deploy/auto_scaler/auto_scaling_group_config.rb

@@ -7,7 +7,7 @@ require "ecs_deploy/auto_scaler/cluster_resource_manager"
 
 module EcsDeploy
   module AutoScaler
-    AutoScalingGroupConfig = Struct.new(:name, :region, :cluster, :buffer, :service_configs) do
+    AutoScalingGroupConfig = Struct.new(:name, :region, :cluster, :buffer, :service_configs, :disable_draining) do
       include ConfigBase
 
       MAX_DETACHABLE_INSTANCE_COUNT = 20
@@ -82,7 +82,7 @@ module EcsDeploy
       def decrease_desired_capacity(count)
         container_instance_arns_in_service = cluster_resource_manager.fetch_container_instance_arns_in_service
         container_instances_in_cluster = cluster_resource_manager.fetch_container_instances_in_cluster
-        auto_scaling_group_instances = instances(reload: true)
+        auto_scaling_group_instances = describe_detachable_instances
         deregisterable_instances = container_instances_in_cluster.select do |i|
           i.pending_tasks_count == 0 &&
             !running_essential_task?(i, container_instance_arns_in_service) &&
@@ -144,15 +144,8 @@ module EcsDeploy
 
       def detach_and_terminate_orphan_instances
         container_instance_ids = cluster_resource_manager.fetch_container_instances_in_cluster.map(&:ec2_instance_id)
-        orphans = instances(reload: true).reject do |i|
+        orphans = describe_detachable_instances.reject do |i|
           next true if container_instance_ids.include?(i.instance_id)
-
-          # The lifecycle state of terminated instances becomes "Terminating", "Terminating:Wait", or "Terminating:Proceed",
-          # and we can't detach instances in such a state.
-          if i.lifecycle_state.start_with?("Terminating")
-            AutoScaler.error_logger.warn("#{log_prefix} The lifesycle state of #{i.instance_id} is \"#{i.lifecycle_state}\", so ignore it")
-            next true
-          end
         end.map(&:instance_id)
 
         return if orphans.empty?
@@ -184,14 +177,11 @@ module EcsDeploy
         )
       end
 
-      def instances(reload: false)
-        if reload || @instances.nil?
-          resp = client.describe_auto_scaling_groups({
-            auto_scaling_group_names: [name],
-          })
-          @instances = resp.auto_scaling_groups[0].instances
-        else
-          @instances
+      def describe_detachable_instances
+        client.describe_auto_scaling_groups({ auto_scaling_group_names: [name] }).auto_scaling_groups[0].instances.reject do |i|
+          # The lifecycle state of terminated instances becomes "Detaching", "Terminating", "Terminating:Wait", or "Terminating:Proceed",
+          # and we can't detach instances in such a state.
+          i.lifecycle_state.start_with?("Terminating") || i.lifecycle_state == "Detaching"
         end
       end
 

data/lib/ecs_deploy/auto_scaler/cluster_resource_manager.rb

@@ -74,39 +74,42 @@ module EcsDeploy
       end
 
       def trigger_capacity_update(old_desired_capacity, new_desired_capacity, interval: 5, wait_until_capacity_updated: false)
+        return if new_desired_capacity == old_desired_capacity
+
         th = Thread.new do
           @logger&.info "#{log_prefix} Start updating capacity: #{old_desired_capacity} -> #{new_desired_capacity}"
           Timeout.timeout(180) do
-            until @capacity == new_desired_capacity || (new_desired_capacity >= old_desired_capacity && @capacity > new_desired_capacity)
+            until @capacity == new_desired_capacity ||
+                (new_desired_capacity > old_desired_capacity && @capacity > new_desired_capacity) ||
+                (new_desired_capacity < old_desired_capacity && @capacity < new_desired_capacity)
               @mutex.synchronize do
-                begin
-                  @capacity = calculate_active_instance_capacity
-                  @resource.broadcast
-                rescue => e
-                  AutoScaler.error_logger.warn("#{log_prefix} `#{__method__}': #{e} (#{e.class})")
-                end
+                @capacity = calculate_active_instance_capacity
+                @resource.broadcast
+              rescue => e
+                AutoScaler.error_logger.warn("#{log_prefix} `#{__method__}': #{e} (#{e.class})")
               end
 
               sleep interval
             end
             @logger&.info "#{log_prefix} capacity is updated to #{@capacity}"
           end
+        rescue Timeout::Error => e
+          msg = "#{log_prefix} `#{__method__}': #{e} (#{e.class})"
+          if @capacity_based_on == "vCPUs"
+            # Timeout::Error sometimes occur.
+            # For example, the following case never meats the condition of until
+            #   * old_desired_capaacity is 102
+            #   * new_desired_capaacity is 101
+            #   * all instances have 2 vCPUs
+            AutoScaler.error_logger.warn(msg)
+          else
+            AutoScaler.error_logger.error(msg)
+          end
         end
 
         if wait_until_capacity_updated
           @logger&.info "#{log_prefix} Wait for the capacity of active instances to become #{new_desired_capacity} from #{old_desired_capacity}"
-          begin
-            th.join
-          rescue Timeout::Error => e
-            msg = "#{log_prefix} `#{__method__}': #{e} (#{e.class})"
-            if @capacity_based_on == "vCPUs"
-              # Timeout::Error sometimes occur.
-              # For example, @capacity won't be new_desired_capacity if new_desired_capacity is odd and all instances have 2 vCPUs
-              AutoScaler.error_logger.warn(msg)
-            else
-              AutoScaler.error_logger.error(msg)
-            end
-          end
+          th.join
         end
       end
 

data/lib/ecs_deploy/auto_scaler/instance_drainer.rb

@@ -78,6 +78,11 @@ module EcsDeploy
       def set_instance_state_to_draining(config_to_instance_ids, region)
         cl = ecs_client(region)
         config_to_instance_ids.each do |config, instance_ids|
+          if config.disable_draining == true || config.disable_draining == "true"
+            @logger.info "Skip draining instances: region: #{region}, cluster: #{config.cluster}, instance_ids: #{instance_ids.inspect}"
+            next
+          end
+
           arns = cl.list_container_instances(
             cluster: config.cluster,
             filter: "ec2InstanceId in [#{instance_ids.join(",")}]",

data/lib/ecs_deploy/auto_scaler/service_config.rb

@@ -135,6 +135,9 @@ module EcsDeploy
           now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           @reach_max_at ||= now
           @logger.info "#{log_prefix} Service waits cooldown elapsed #{(now - @reach_max_at).to_i}sec"
+          if next_desired_count > max_task_count[current_level] && current_level == max_task_count.size - 1
+            @logger.warn "#{log_prefix} Desired count has reached the maximum value and couldn't be increased"
+          end
         elsif current_level == next_level && next_desired_count < max_task_count[current_level]
           level = current_level
           @reach_max_at = nil

data/lib/ecs_deploy/auto_scaler/spot_fleet_request_config.rb

@@ -8,7 +8,7 @@ require "ecs_deploy/auto_scaler/cluster_resource_manager"
 
 module EcsDeploy
   module AutoScaler
-    SpotFleetRequestConfig = Struct.new(:id, :region, :cluster, :buffer, :service_configs) do
+    SpotFleetRequestConfig = Struct.new(:id, :region, :cluster, :buffer, :service_configs, :disable_draining) do
       include ConfigBase
 
       def initialize(attributes = {}, logger)

data/lib/ecs_deploy/capistrano.rb

@@ -113,6 +113,7 @@ namespace :ecs do
           service_options[:deployment_configuration] = service[:deployment_configuration] if service[:deployment_configuration]
           service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
           service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
+          service_options[:capacity_provider_strategy] = service[:capacity_provider_strategy] if service[:capacity_provider_strategy]
           service_options[:scheduling_strategy] = service[:scheduling_strategy] if service[:scheduling_strategy]
           s = EcsDeploy::Service.new(**service_options)
           s.deploy
@@ -179,6 +180,7 @@ namespace :ecs do
           service_options[:deployment_configuration] = service[:deployment_configuration] if service[:deployment_configuration]
           service_options[:placement_constraints] = service[:placement_constraints] if service[:placement_constraints]
           service_options[:placement_strategy] = service[:placement_strategy] if service[:placement_strategy]
+          service_options[:capacity_provider_strategy] = service[:capacity_provider_strategy] if service[:capacity_provider_strategy]
           s = EcsDeploy::Service.new(**service_options)
           s.deploy
           EcsDeploy::TaskDefinition.deregister(current_task_definition_arn, region: r)

data/lib/ecs_deploy/instance_fluctuation_manager.rb

@@ -56,6 +56,9 @@ module EcsDeploy
         ).container_instances
       end
 
+      # The status of ECS instances sometimes seems to remain 'DEREGISTERING' for a few minutes after they are terminated.
+      container_instances.reject! { |ci| ci.status == 'DEREGISTERING' }
+
       az_to_container_instances = container_instances.sort_by {|ci| - ci.running_tasks_count }.group_by do |ci|
         ci.attributes.find {|attribute| attribute.name == "ecs.availability-zone" }.value
       end

data/lib/ecs_deploy/service.rb

@@ -7,7 +7,7 @@ module EcsDeploy
 
     class TooManyAttemptsError < StandardError; end
 
-    attr_reader :cluster, :region, :service_name, :delete
+    attr_reader :cluster, :region, :service_name, :delete, :deploy_started_at
 
     def initialize(
       cluster:, service_name:, task_definition_name: nil, revision: nil,
@@ -16,6 +16,7 @@ module EcsDeploy
       launch_type: nil,
       placement_constraints: [],
       placement_strategy: [],
+      capacity_provider_strategy: nil,
       network_configuration: nil,
       health_check_grace_period_seconds: nil,
       scheduling_strategy: 'REPLICA',
@@ -35,6 +36,7 @@ module EcsDeploy
       @launch_type = launch_type
       @placement_constraints = placement_constraints
       @placement_strategy = placement_strategy
+      @capacity_provider_strategy = capacity_provider_strategy
       @network_configuration = network_configuration
       @health_check_grace_period_seconds = health_check_grace_period_seconds
       @scheduling_strategy = scheduling_strategy
@@ -59,6 +61,7 @@ module EcsDeploy
     end
 
     def deploy
+      @deploy_started_at = Time.now
       res = @client.describe_services(cluster: @cluster, services: [@service_name])
       service_options = {
         cluster: @cluster,
@@ -66,8 +69,25 @@ module EcsDeploy
         deployment_configuration: @deployment_configuration,
         network_configuration: @network_configuration,
         health_check_grace_period_seconds: @health_check_grace_period_seconds,
+        capacity_provider_strategy: @capacity_provider_strategy,
         enable_execute_command: @enable_execute_command,
+        enable_ecs_managed_tags: @enable_ecs_managed_tags,
+        placement_constraints: @placement_constraints,
+        placement_strategy: @placement_strategy,
       }
+
+      if @load_balancers && EcsDeploy.config.ecs_service_role
+        service_options.merge!({
+          role: EcsDeploy.config.ecs_service_role,
+        })
+      end
+
+      if @load_balancers
+        service_options.merge!({
+          load_balancers: @load_balancers,
+        })
+      end
+
       if res.services.select{ |s| s.status == 'ACTIVE' }.empty?
         return if @delete
 
@@ -75,49 +95,56 @@ module EcsDeploy
           service_name: @service_name,
           desired_count: @desired_count.to_i,
           launch_type: @launch_type,
-          placement_constraints: @placement_constraints,
-          placement_strategy: @placement_strategy,
-          enable_ecs_managed_tags: @enable_ecs_managed_tags,
           tags: @tags,
           propagate_tags: @propagate_tags,
         })
 
-        if @load_balancers && EcsDeploy.config.ecs_service_role
-          service_options.merge!({
-            role: EcsDeploy.config.ecs_service_role,
-          })
-        end
-
-        if @load_balancers
-          service_options.merge!({
-            load_balancers: @load_balancers,
-          })
-        end
-
         if @scheduling_strategy == 'DAEMON'
           service_options[:scheduling_strategy] = @scheduling_strategy
           service_options.delete(:desired_count)
         end
         @response = @client.create_service(service_options)
-        EcsDeploy.logger.info "create service [#{@service_name}] [#{@region}] [#{Paint['OK', :green]}]"
+        EcsDeploy.logger.info "create service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
       else
         return delete_service if @delete
 
         service_options.merge!({service: @service_name})
         service_options.merge!({desired_count: @desired_count}) if @desired_count
+        service_options.merge!({propagate_tags: @propagate_tags}) if @propagate_tags
+
+        current_service = res.services[0]
+        service_options.merge!({force_new_deployment: true}) if need_force_new_deployment?(current_service)
+
         update_tags(@service_name, @tags)
         @response = @client.update_service(service_options)
-        EcsDeploy.logger.info "update service [#{@service_name}] [#{@region}] [#{Paint['OK', :green]}]"
+        EcsDeploy.logger.info "update service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
       end
     end
 
+    private def need_force_new_deployment?(service)
+      return false unless @capacity_provider_strategy
+      return true unless service.capacity_provider_strategy
+
+      return true if @capacity_provider_strategy.size != service.capacity_provider_strategy.size
+
+      match_array = @capacity_provider_strategy.all? do |strategy|
+        service.capacity_provider_strategy.find do |current_strategy|
+          strategy[:capacity_provider] == current_strategy.capacity_provider &&
+            strategy[:weight] == current_strategy.weight &&
+            strategy[:base] == current_strategy.base
+        end
+      end
+
+      !match_array
+    end
+
     def delete_service
       if @scheduling_strategy != 'DAEMON'
         @client.update_service(cluster: @cluster, service: @service_name, desired_count: 0)
         sleep 1
       end
       @client.delete_service(cluster: @cluster, service: @service_name)
-      EcsDeploy.logger.info "delete service [#{@service_name}] [#{@region}] [#{Paint['OK', :green]}]"
+      EcsDeploy.logger.info "delete service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
     end
 
     def update_tags(service_name, tags)
@@ -142,19 +169,31 @@ module EcsDeploy
       end
     end
 
+    def log_events(ecs_service)
+      ecs_service.events.sort_by(&:created_at).each do |e|
+        next if e.created_at <= deploy_started_at
+        next if @last_event && e.created_at <= @last_event.created_at
+
+        EcsDeploy.logger.info e.message
+        @last_event = e
+      end
+    end
+
     def self.wait_all_running(services)
       services.group_by { |s| [s.cluster, s.region] }.flat_map do |(cl, region), ss|
         client = Aws::ECS::Client.new(region: region)
         ss.reject(&:delete).map(&:service_name).each_slice(MAX_DESCRIBE_SERVICES).map do |chunked_service_names|
           Thread.new do
             EcsDeploy.config.ecs_wait_until_services_stable_max_attempts.times do
-              EcsDeploy.logger.info "wait service stable [#{chunked_service_names.join(", ")}]"
+              EcsDeploy.logger.info "wait service stable [#{chunked_service_names.join(", ")}] [#{cl}]"
               resp = client.describe_services(cluster: cl, services: chunked_service_names)
               resp.services.each do |s|
                 # cf. https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-ecs/lib/aws-sdk-ecs/waiters.rb#L91-L96
                 if s.deployments.size == 1 && s.running_count == s.desired_count
                   chunked_service_names.delete(s.service_name)
                 end
+                service = ss.detect {|sc| sc.service_name == s.service_name }
+                service.log_events(s)
               end
               break if chunked_service_names.empty?
               sleep EcsDeploy.config.ecs_wait_until_services_stable_delay

data/lib/ecs_deploy/version.rb

@@ -1,3 +1,3 @@
 module EcsDeploy
-  VERSION = "1.0.2"
+  VERSION = "1.0.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ecs_deploy
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.4
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-26 00:00:00.000000000 Z
+date: 2023-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-autoscaling
@@ -170,6 +170,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: AWS ECS deploy helper
 email:
 - kakyoin.hierophant@gmail.com
@@ -178,8 +192,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - README.md
@@ -222,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.4.2
 signing_key:
 specification_version: 4
 summary: AWS ECS deploy helper

data/.travis.yml

@@ -1,5 +0,0 @@
-language: ruby
-rvm:
-  - ruby-2.4.9
-  - ruby-2.5.7
-  - ruby-2.6.5