ecs_deploy 0.3.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,198 @@
1
+ require "aws-sdk-autoscaling"
2
+ require "aws-sdk-ec2"
3
+ require "aws-sdk-ecs"
4
+
5
+ module EcsDeploy
6
+ class InstanceFluctuationManager
7
+ attr_reader :logger
8
+
9
+ MAX_UPDATABLE_ECS_CONTAINER_COUNT = 10
10
+ MAX_DETACHEABLE_EC2_INSTACE_COUNT = 20
11
+ MAX_DESCRIBABLE_ECS_TASK_COUNT = 100
12
+
13
+ def initialize(region:, cluster:, auto_scaling_group_name:, desired_capacity:, logger:)
14
+ @region = region
15
+ @cluster = cluster
16
+ @auto_scaling_group_name = auto_scaling_group_name
17
+ @desired_capacity = desired_capacity
18
+ @logger = logger
19
+ end
20
+
21
+ def increase
22
+ asg = fetch_auto_scaling_group
23
+
24
+ @logger.info("Increase desired capacity of #{@auto_scaling_group_name}: #{asg.desired_capacity} => #{asg.max_size}")
25
+ as_client.update_auto_scaling_group(auto_scaling_group_name: @auto_scaling_group_name, desired_capacity: asg.max_size)
26
+
27
+ # Run in background because increasing instances may take time
28
+ Thread.new do
29
+ loop do
30
+ cluster = ecs_client.describe_clusters(clusters: [@cluster]).clusters.first
31
+ instance_count = cluster.registered_container_instances_count
32
+ if instance_count == asg.max_size
33
+ @logger.info("Succeeded in increasing instances!")
34
+ break
35
+ end
36
+ @logger.info("Current registered instance count: #{instance_count}")
37
+ sleep 5
38
+ end
39
+ end
40
+ end
41
+
42
+ def decrease
43
+ asg = fetch_auto_scaling_group
44
+
45
+ decrease_count = asg.desired_capacity - @desired_capacity
46
+ if decrease_count <= 0
47
+ @logger.info("The capacity is already #{asg.desired_capacity}")
48
+ return
49
+ end
50
+ @logger.info("Decrease desired capacity of #{@auto_scaling_group_name}: #{asg.desired_capacity} => #{@desired_capacity}")
51
+
52
+ container_instances = ecs_client.list_container_instances(cluster: @cluster).flat_map do |resp|
53
+ ecs_client.describe_container_instances(
54
+ cluster: @cluster,
55
+ container_instances: resp.container_instance_arns
56
+ ).container_instances
57
+ end
58
+
59
+ # The status of ECS instances sometimes seems to remain 'DEREGISTERING' for a few minutes after they are terminated.
60
+ container_instances.reject! { |ci| ci.status == 'DEREGISTERING' }
61
+
62
+ az_to_container_instances = container_instances.sort_by {|ci| - ci.running_tasks_count }.group_by do |ci|
63
+ ci.attributes.find {|attribute| attribute.name == "ecs.availability-zone" }.value
64
+ end
65
+ if az_to_container_instances.empty?
66
+ @logger.info("There are no instances to terminate.")
67
+ return
68
+ end
69
+
70
+ target_container_instances = extract_target_container_instances(decrease_count, az_to_container_instances)
71
+
72
+ @logger.info("running tasks: #{ecs_client.list_tasks(cluster: @cluster).task_arns.size}")
73
+ all_running_task_arns = []
74
+ target_container_instances.map(&:container_instance_arn).each_slice(MAX_UPDATABLE_ECS_CONTAINER_COUNT) do |arns|
75
+ @logger.info(arns)
76
+ ecs_client.update_container_instances_state(
77
+ cluster: @cluster,
78
+ container_instances: arns,
79
+ status: "DRAINING"
80
+ )
81
+ arns.each do |arn|
82
+ all_running_task_arns.concat(list_running_task_arns(arn))
83
+ end
84
+ end
85
+
86
+ stop_tasks_not_belonging_service(all_running_task_arns)
87
+ wait_until_tasks_stopped(all_running_task_arns)
88
+
89
+ instance_ids = target_container_instances.map(&:ec2_instance_id)
90
+ terminate_instances(instance_ids)
91
+ @logger.info("Succeeded in decreasing instances!")
92
+ end
93
+
94
+ private
95
+
96
+ def aws_params
97
+ {
98
+ access_key_id: EcsDeploy.config.access_key_id,
99
+ secret_access_key: EcsDeploy.config.secret_access_key,
100
+ region: @region,
101
+ logger: @logger
102
+ }.reject do |_key, value|
103
+ value.nil?
104
+ end
105
+ end
106
+
107
+ def as_client
108
+ @as_client ||= Aws::AutoScaling::Client.new(aws_params)
109
+ end
110
+
111
+ def ec2_client
112
+ @ec2_client ||= Aws::EC2::Client.new(aws_params)
113
+ end
114
+
115
+ def ecs_client
116
+ @ecs_client ||= Aws::ECS::Client.new(aws_params)
117
+ end
118
+
119
+ def fetch_auto_scaling_group
120
+ as_client.describe_auto_scaling_groups(auto_scaling_group_names: [@auto_scaling_group_name]).auto_scaling_groups.first
121
+ end
122
+
123
+ # Extract container instances to terminate considering AZ balance
124
+ def extract_target_container_instances(decrease_count, az_to_container_instances)
125
+ target_container_instances = []
126
+ decrease_count.times do
127
+ @logger.debug do
128
+ "AZ balance: #{az_to_container_instances.sort_by {|az, _| az }.map {|az, instances| [az, instances.size] }.to_h}"
129
+ end
130
+ az = az_to_container_instances.max_by {|_az, instances| instances.size }.first
131
+ target_container_instances << az_to_container_instances[az].pop
132
+ end
133
+ @logger.info do
134
+ "AZ balance: #{az_to_container_instances.sort_by {|az, _| az }.map {|az, instances| [az, instances.size] }.to_h}"
135
+ end
136
+
137
+ target_container_instances
138
+ end
139
+
140
+ # list tasks whose desired_status is "RUNNING" or
141
+ # whoose desired_status is "STOPPED" but last_status is "RUNNING" on the ECS container
142
+ def list_running_task_arns(container_instance_arn)
143
+ running_tasks_arn = ecs_client.list_tasks(cluster: @cluster, container_instance: container_instance_arn).flat_map(&:task_arns)
144
+ stopped_tasks_arn = ecs_client.list_tasks(cluster: @cluster, container_instance: container_instance_arn, desired_status: "STOPPED").flat_map(&:task_arns)
145
+ stopped_running_task_arns = stopped_tasks_arn.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).flat_map do |arns|
146
+ ecs_client.describe_tasks(cluster: @cluster, tasks: arns).tasks.select do |task|
147
+ task.desired_status == "STOPPED" && task.last_status == "RUNNING"
148
+ end
149
+ end.map(&:task_arn)
150
+ running_tasks_arn + stopped_running_task_arns
151
+ end
152
+
153
+ def wait_until_tasks_stopped(task_arns)
154
+ @logger.info("All old tasks: #{task_arns.size}")
155
+ task_arns.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).each do |arns|
156
+ ecs_client.wait_until(:tasks_stopped, cluster: @cluster, tasks: arns)
157
+ end
158
+ @logger.info("All old tasks are stopped")
159
+ end
160
+
161
+ def stop_tasks_not_belonging_service(running_task_arns)
162
+ @logger.info("Running tasks: #{running_task_arns.size}")
163
+ unless running_task_arns.empty?
164
+ running_task_arns.each_slice(MAX_DESCRIBABLE_ECS_TASK_COUNT).each do |arns|
165
+ ecs_client.describe_tasks(cluster: @cluster, tasks: arns).tasks.each do |task|
166
+ ecs_client.stop_task(cluster: @cluster, task: task.task_arn) if task.group.start_with?("family:")
167
+ end
168
+ end
169
+ end
170
+ end
171
+
172
+ def terminate_instances(instance_ids)
173
+ if instance_ids.empty?
174
+ @logger.info("There are no instances to terminate.")
175
+ return
176
+ end
177
+ instance_ids.each_slice(MAX_DETACHEABLE_EC2_INSTACE_COUNT) do |ids|
178
+ as_client.detach_instances(
179
+ auto_scaling_group_name: @auto_scaling_group_name,
180
+ instance_ids: ids,
181
+ should_decrement_desired_capacity: true
182
+ )
183
+ end
184
+
185
+ ec2_client.terminate_instances(instance_ids: instance_ids)
186
+
187
+ ec2_client.wait_until(:instance_terminated, instance_ids: instance_ids) do |w|
188
+ w.before_wait do |attempts, response|
189
+ @logger.info("Waiting for stopping all instances...#{attempts}")
190
+ instances = response.reservations.flat_map(&:instances)
191
+ instances.sort_by(&:instance_id).each do |instance|
192
+ @logger.info("#{instance.instance_id}\t#{instance.state.name}")
193
+ end
194
+ end
195
+ end
196
+ end
197
+ end
198
+ end
@@ -1,3 +1,4 @@
1
+ require 'aws-sdk-cloudwatchevents'
1
2
  require 'timeout'
2
3
 
3
4
  module EcsDeploy
@@ -8,7 +9,7 @@ module EcsDeploy
8
9
 
9
10
  def initialize(
10
11
  cluster:, rule_name:, schedule_expression:, enabled: true, description: nil, target_id: nil,
11
- task_definition_name:, revision: nil, task_count: nil, role_arn:,
12
+ task_definition_name:, revision: nil, task_count: nil, role_arn:, network_configuration: nil, launch_type: nil, platform_version: nil, group: nil,
12
13
  region: nil, container_overrides: nil
13
14
  )
14
15
  @cluster = cluster
@@ -21,10 +22,15 @@ module EcsDeploy
21
22
  @task_count = task_count || 1
22
23
  @revision = revision
23
24
  @role_arn = role_arn
24
- @region = region || EcsDeploy.config.default_region || ENV["AWS_DEFAULT_REGION"]
25
+ @network_configuration = network_configuration
26
+ @launch_type = launch_type || "EC2"
27
+ @platform_version = platform_version
28
+ @group = group
29
+ region ||= EcsDeploy.config.default_region
25
30
  @container_overrides = container_overrides
26
31
 
27
- @client = Aws::ECS::Client.new(region: @region)
32
+ @client = region ? Aws::ECS::Client.new(region: region) : Aws::ECS::Client.new
33
+ @region = @client.config.region
28
34
  @cloud_watch_events = Aws::CloudWatchEvents::Client.new(region: @region)
29
35
  end
30
36
 
@@ -66,8 +72,14 @@ module EcsDeploy
66
72
  ecs_parameters: {
67
73
  task_definition_arn: task_definition_arn,
68
74
  task_count: @task_count,
75
+ network_configuration: @network_configuration,
76
+ launch_type: @launch_type,
77
+ platform_version: @platform_version,
78
+ group: @group,
69
79
  },
70
80
  }
81
+ target[:ecs_parameters].compact!
82
+
71
83
  if @container_overrides
72
84
  target.merge!(input: { containerOverrides: @container_overrides }.to_json)
73
85
  end
@@ -5,13 +5,26 @@ module EcsDeploy
5
5
  CHECK_INTERVAL = 5
6
6
  MAX_DESCRIBE_SERVICES = 10
7
7
 
8
- attr_reader :cluster, :region, :service_name
8
+ class TooManyAttemptsError < StandardError; end
9
+
10
+ attr_reader :cluster, :region, :service_name, :delete
9
11
 
10
12
  def initialize(
11
13
  cluster:, service_name:, task_definition_name: nil, revision: nil,
12
14
  load_balancers: nil,
13
15
  desired_count: nil, deployment_configuration: {maximum_percent: 200, minimum_healthy_percent: 100},
14
- region: nil
16
+ launch_type: nil,
17
+ placement_constraints: [],
18
+ placement_strategy: [],
19
+ network_configuration: nil,
20
+ health_check_grace_period_seconds: nil,
21
+ scheduling_strategy: 'REPLICA',
22
+ enable_ecs_managed_tags: nil,
23
+ tags: nil,
24
+ propagate_tags: nil,
25
+ region: nil,
26
+ delete: false,
27
+ enable_execute_command: false
15
28
  )
16
29
  @cluster = cluster
17
30
  @service_name = service_name
@@ -19,11 +32,25 @@ module EcsDeploy
19
32
  @load_balancers = load_balancers
20
33
  @desired_count = desired_count
21
34
  @deployment_configuration = deployment_configuration
35
+ @launch_type = launch_type
36
+ @placement_constraints = placement_constraints
37
+ @placement_strategy = placement_strategy
38
+ @network_configuration = network_configuration
39
+ @health_check_grace_period_seconds = health_check_grace_period_seconds
40
+ @scheduling_strategy = scheduling_strategy
22
41
  @revision = revision
23
- @region = region || EcsDeploy.config.default_region || ENV["AWS_DEFAULT_REGION"]
42
+ @enable_ecs_managed_tags = enable_ecs_managed_tags
43
+ @tags = tags
44
+ @propagate_tags = propagate_tags
45
+ @enable_execute_command = enable_execute_command
46
+
24
47
  @response = nil
25
48
 
26
- @client = Aws::ECS::Client.new(region: @region)
49
+ region ||= EcsDeploy.config.default_region
50
+ @client = region ? Aws::ECS::Client.new(region: region) : Aws::ECS::Client.new
51
+ @region = @client.config.region
52
+
53
+ @delete = delete
27
54
  end
28
55
 
29
56
  def current_task_definition_arn
@@ -37,53 +64,105 @@ module EcsDeploy
37
64
  cluster: @cluster,
38
65
  task_definition: task_definition_name_with_revision,
39
66
  deployment_configuration: @deployment_configuration,
67
+ network_configuration: @network_configuration,
68
+ health_check_grace_period_seconds: @health_check_grace_period_seconds,
69
+ enable_execute_command: @enable_execute_command,
40
70
  }
41
71
  if res.services.select{ |s| s.status == 'ACTIVE' }.empty?
72
+ return if @delete
73
+
42
74
  service_options.merge!({
43
75
  service_name: @service_name,
44
76
  desired_count: @desired_count.to_i,
77
+ launch_type: @launch_type,
78
+ placement_constraints: @placement_constraints,
79
+ placement_strategy: @placement_strategy,
80
+ enable_ecs_managed_tags: @enable_ecs_managed_tags,
81
+ tags: @tags,
82
+ propagate_tags: @propagate_tags,
45
83
  })
46
- if @load_balancers
84
+
85
+ if @load_balancers && EcsDeploy.config.ecs_service_role
47
86
  service_options.merge!({
48
87
  role: EcsDeploy.config.ecs_service_role,
88
+ })
89
+ end
90
+
91
+ if @load_balancers
92
+ service_options.merge!({
49
93
  load_balancers: @load_balancers,
50
94
  })
51
95
  end
96
+
97
+ if @scheduling_strategy == 'DAEMON'
98
+ service_options[:scheduling_strategy] = @scheduling_strategy
99
+ service_options.delete(:desired_count)
100
+ end
52
101
  @response = @client.create_service(service_options)
53
- EcsDeploy.logger.info "create service [#{@service_name}] [#{@region}] [#{Paint['OK', :green]}]"
102
+ EcsDeploy.logger.info "create service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
54
103
  else
104
+ return delete_service if @delete
105
+
55
106
  service_options.merge!({service: @service_name})
56
107
  service_options.merge!({desired_count: @desired_count}) if @desired_count
108
+ update_tags(@service_name, @tags)
57
109
  @response = @client.update_service(service_options)
58
- EcsDeploy.logger.info "update service [#{@service_name}] [#{@region}] [#{Paint['OK', :green]}]"
110
+ EcsDeploy.logger.info "update service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
59
111
  end
60
112
  end
61
113
 
62
- def wait_running
63
- return if @response.nil?
114
+ def delete_service
115
+ if @scheduling_strategy != 'DAEMON'
116
+ @client.update_service(cluster: @cluster, service: @service_name, desired_count: 0)
117
+ sleep 1
118
+ end
119
+ @client.delete_service(cluster: @cluster, service: @service_name)
120
+ EcsDeploy.logger.info "delete service [#{@service_name}] [#{@cluster}] [#{@region}] [#{Paint['OK', :green]}]"
121
+ end
64
122
 
65
- service = @response.service
123
+ def update_tags(service_name, tags)
124
+ service_arn = @client.describe_services(cluster: @cluster, services: [service_name]).services.first.service_arn
125
+ if service_arn.split('/').size == 2
126
+ if tags
127
+ EcsDeploy.logger.warn "#{service_name} doesn't support tagging operations, so tags are ignored. Long arn format must be used for tagging operations."
128
+ end
129
+ return
130
+ end
66
131
 
67
- @client.wait_until(:services_stable, cluster: @cluster, services: [service.service_name]) do |w|
68
- w.delay = 10
132
+ tags ||= []
133
+ current_tag_keys = @client.list_tags_for_resource(resource_arn: service_arn).tags.map(&:key)
134
+ deleted_tag_keys = current_tag_keys - tags.map { |t| t[:key] }
69
135
 
70
- w.before_attempt do
71
- EcsDeploy.logger.info "wait service stable [#{service.service_name}]"
72
- end
136
+ unless deleted_tag_keys.empty?
137
+ @client.untag_resource(resource_arn: service_arn, tag_keys: deleted_tag_keys)
138
+ end
139
+
140
+ unless tags.empty?
141
+ @client.tag_resource(resource_arn: service_arn, tags: tags)
73
142
  end
74
143
  end
75
144
 
76
145
  def self.wait_all_running(services)
77
- services.group_by { |s| [s.cluster, s.region] }.each do |(cl, region), ss|
146
+ services.group_by { |s| [s.cluster, s.region] }.flat_map do |(cl, region), ss|
78
147
  client = Aws::ECS::Client.new(region: region)
79
- ss.map(&:service_name).each_slice(MAX_DESCRIBE_SERVICES) do |chunked_service_names|
80
- client.wait_until(:services_stable, cluster: cl, services: chunked_service_names) do |w|
81
- w.before_attempt do
82
- EcsDeploy.logger.info "wait service stable [#{chunked_service_names.join(", ")}]"
148
+ ss.reject(&:delete).map(&:service_name).each_slice(MAX_DESCRIBE_SERVICES).map do |chunked_service_names|
149
+ Thread.new do
150
+ EcsDeploy.config.ecs_wait_until_services_stable_max_attempts.times do
151
+ EcsDeploy.logger.info "wait service stable [#{chunked_service_names.join(", ")}] [#{cl}]"
152
+ resp = client.describe_services(cluster: cl, services: chunked_service_names)
153
+ resp.services.each do |s|
154
+ # cf. https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-ecs/lib/aws-sdk-ecs/waiters.rb#L91-L96
155
+ if s.deployments.size == 1 && s.running_count == s.desired_count
156
+ chunked_service_names.delete(s.service_name)
157
+ end
158
+ end
159
+ break if chunked_service_names.empty?
160
+ sleep EcsDeploy.config.ecs_wait_until_services_stable_delay
83
161
  end
162
+ raise TooManyAttemptsError unless chunked_service_names.empty?
84
163
  end
85
164
  end
86
- end
165
+ end.each(&:join)
87
166
  end
88
167
 
89
168
  private
@@ -1,37 +1,54 @@
1
1
  module EcsDeploy
2
2
  class TaskDefinition
3
+ RETRY_BACKOFF = lambda do |c|
4
+ sleep(1)
5
+ end
6
+
7
+ RETRY_LIMIT = 10
8
+
3
9
  def self.deregister(arn, region: nil)
4
- region = region || EcsDeploy.config.default_region || ENV["AWS_DEFAULT_REGION"]
5
- client = Aws::ECS::Client.new(region: region)
10
+ region ||= EcsDeploy.config.default_region
11
+ param = {retry_backoff: RETRY_BACKOFF, retry_limit: RETRY_LIMIT}
12
+ client = region ? Aws::ECS::Client.new(param.merge(region: region)) : Aws::ECS::Client.new(param)
6
13
  client.deregister_task_definition({
7
14
  task_definition: arn,
8
15
  })
9
- EcsDeploy.logger.info "deregister task definition [#{arn}] [#{region}] [#{Paint['OK', :green]}]"
16
+ EcsDeploy.logger.info "deregister task definition [#{arn}] [#{client.config.region}] [#{Paint['OK', :green]}]"
10
17
  end
11
18
 
12
19
  def initialize(
13
20
  task_definition_name:, region: nil,
14
21
  network_mode: "bridge", volumes: [], container_definitions: [], placement_constraints: [],
15
- task_role_arn: nil
22
+ task_role_arn: nil,
23
+ execution_role_arn: nil,
24
+ requires_compatibilities: nil,
25
+ cpu: nil, memory: nil,
26
+ tags: nil
16
27
  )
17
28
  @task_definition_name = task_definition_name
18
29
  @task_role_arn = task_role_arn
19
- @region = region || EcsDeploy.config.default_region || ENV["AWS_DEFAULT_REGION"]
30
+ @execution_role_arn = execution_role_arn
31
+ region ||= EcsDeploy.config.default_region
20
32
 
21
33
  @container_definitions = container_definitions.map do |cd|
22
34
  if cd[:docker_labels]
23
35
  cd[:docker_labels] = cd[:docker_labels].map { |k, v| [k.to_s, v] }.to_h
24
36
  end
25
- if cd[:log_configuration] && cd[:log_configuration][:options]
26
- cd[:log_configuration][:options] = cd[:log_configuration][:options].map { |k, v| [k.to_s, v] }.to_h
37
+ if cd.dig(:log_configuration, :options)
38
+ cd[:log_configuration][:options] = cd.dig(:log_configuration, :options).map { |k, v| [k.to_s, v] }.to_h
27
39
  end
28
40
  cd
29
41
  end
30
42
  @volumes = volumes
31
43
  @network_mode = network_mode
32
44
  @placement_constraints = placement_constraints
33
-
34
- @client = Aws::ECS::Client.new(region: @region)
45
+ @requires_compatibilities = requires_compatibilities
46
+ @cpu = cpu&.to_s
47
+ @memory = memory&.to_s
48
+ @tags = tags
49
+ param = {retry_backoff: RETRY_BACKOFF, retry_limit: RETRY_LIMIT}
50
+ @client = region ? Aws::ECS::Client.new(param.merge(region: region)) : Aws::ECS::Client.new(param)
51
+ @region = @client.config.region
35
52
  end
36
53
 
37
54
  def recent_task_definition_arns
@@ -52,6 +69,10 @@ module EcsDeploy
52
69
  volumes: @volumes,
53
70
  placement_constraints: @placement_constraints,
54
71
  task_role_arn: @task_role_arn,
72
+ execution_role_arn: @execution_role_arn,
73
+ requires_compatibilities: @requires_compatibilities,
74
+ cpu: @cpu, memory: @memory,
75
+ tags: @tags
55
76
  })
56
77
  EcsDeploy.logger.info "register task definition [#{@task_definition_name}] [#{@region}] [#{Paint['OK', :green]}]"
57
78
  res.task_definition
@@ -1,3 +1,3 @@
1
1
  module EcsDeploy
2
- VERSION = "0.3.2"
2
+ VERSION = "1.0.3"
3
3
  end
data/lib/ecs_deploy.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  require "ecs_deploy/version"
2
2
  require "ecs_deploy/configuration"
3
3
 
4
- require 'aws-sdk'
4
+ require 'aws-sdk-ecs'
5
5
  require 'logger'
6
6
  require 'terminal-table'
7
7
  require 'paint'