wrapbox 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +0 -1
- data/.github/workflows/main.yml +30 -0
- data/README.md +94 -30
- data/lib/wrapbox/config_repository.rb +6 -1
- data/lib/wrapbox/configuration.rb +23 -6
- data/lib/wrapbox/log_fetcher/awslogs.rb +25 -13
- data/lib/wrapbox/runner/docker.rb +19 -3
- data/lib/wrapbox/runner/ecs/instance_manager.rb +92 -0
- data/lib/wrapbox/runner/ecs/task_waiter.rb +120 -0
- data/lib/wrapbox/runner/ecs.rb +177 -111
- data/lib/wrapbox/version.rb +1 -1
- data/lib/wrapbox.rb +19 -6
- data/wrapbox.gemspec +5 -3
- metadata +45 -16
- data/.travis.yml +0 -5
data/lib/wrapbox/runner/ecs.rb
CHANGED
|
@@ -4,12 +4,15 @@ require "multi_json"
|
|
|
4
4
|
require "thor"
|
|
5
5
|
require "yaml"
|
|
6
6
|
require "active_support/core_ext/hash"
|
|
7
|
-
require "logger"
|
|
8
7
|
require "pp"
|
|
9
8
|
require "shellwords"
|
|
9
|
+
require "thwait"
|
|
10
10
|
|
|
11
|
+
require "wrapbox"
|
|
11
12
|
require "wrapbox/config_repository"
|
|
12
13
|
require "wrapbox/log_fetcher"
|
|
14
|
+
require "wrapbox/runner/ecs/instance_manager"
|
|
15
|
+
require "wrapbox/runner/ecs/task_waiter"
|
|
13
16
|
require "wrapbox/version"
|
|
14
17
|
|
|
15
18
|
module Wrapbox
|
|
@@ -29,13 +32,11 @@ module Wrapbox
|
|
|
29
32
|
attr_reader \
|
|
30
33
|
:name,
|
|
31
34
|
:revision,
|
|
32
|
-
:cluster,
|
|
33
35
|
:region,
|
|
34
36
|
:container_definitions,
|
|
35
37
|
:volumes,
|
|
36
38
|
:placement_constraints,
|
|
37
39
|
:placement_strategy,
|
|
38
|
-
:launch_type,
|
|
39
40
|
:requires_compatibilities,
|
|
40
41
|
:task_definition_name,
|
|
41
42
|
:main_container_name,
|
|
@@ -43,8 +44,21 @@ module Wrapbox
|
|
|
43
44
|
:network_configuration,
|
|
44
45
|
:cpu,
|
|
45
46
|
:memory,
|
|
46
|
-
:
|
|
47
|
-
:
|
|
47
|
+
:enable_ecs_managed_tags,
|
|
48
|
+
:tags,
|
|
49
|
+
:propagate_tags,
|
|
50
|
+
:enable_execute_command
|
|
51
|
+
|
|
52
|
+
def self.split_overridable_options_and_parameters(options)
|
|
53
|
+
opts = options.dup
|
|
54
|
+
overridable_options = {}
|
|
55
|
+
%i[cluster launch_type task_role_arn execution_role_arn tags propagate_tags].each do |key|
|
|
56
|
+
value = opts.delete(key)
|
|
57
|
+
overridable_options[key] = value if value
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
[overridable_options, opts]
|
|
61
|
+
end
|
|
48
62
|
|
|
49
63
|
def initialize(options)
|
|
50
64
|
@name = options[:name]
|
|
@@ -53,14 +67,23 @@ module Wrapbox
|
|
|
53
67
|
@cluster = options[:cluster]
|
|
54
68
|
@region = options[:region]
|
|
55
69
|
@volumes = options[:volumes]
|
|
56
|
-
@placement_constraints = options[:placement_constraints]
|
|
70
|
+
@placement_constraints = options[:placement_constraints] || []
|
|
57
71
|
@placement_strategy = options[:placement_strategy]
|
|
72
|
+
@capacity_provider_strategy = options[:capacity_provider_strategy] || []
|
|
58
73
|
@launch_type = options[:launch_type]
|
|
59
74
|
@requires_compatibilities = options[:requires_compatibilities]
|
|
60
75
|
@network_mode = options[:network_mode]
|
|
61
76
|
@network_configuration = options[:network_configuration]
|
|
62
77
|
@cpu = options[:cpu]
|
|
63
78
|
@memory = options[:memory]
|
|
79
|
+
@enable_ecs_managed_tags = options[:enable_ecs_managed_tags]
|
|
80
|
+
@tags = options[:tags]
|
|
81
|
+
@propagate_tags = options[:propagate_tags]
|
|
82
|
+
@enable_execute_command = options[:enable_execute_command]
|
|
83
|
+
if options[:launch_instances]
|
|
84
|
+
@instance_manager = Wrapbox::Runner::Ecs::InstanceManager.new(@cluster, @region, **options[:launch_instances])
|
|
85
|
+
end
|
|
86
|
+
@task_waiter = Wrapbox::Runner::Ecs::TaskWaiter.new(cluster: @cluster, region: @region, delay: WAIT_DELAY)
|
|
64
87
|
|
|
65
88
|
@container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions] || []
|
|
66
89
|
@container_definitions.concat(options[:additional_container_definitions]) if options[:additional_container_definitions] # deprecated
|
|
@@ -93,22 +116,17 @@ module Wrapbox
|
|
|
93
116
|
|
|
94
117
|
@task_role_arn = options[:task_role_arn]
|
|
95
118
|
@execution_role_arn = options[:execution_role_arn]
|
|
96
|
-
|
|
97
|
-
@logger = Logger.new($stdout)
|
|
119
|
+
@logger = Wrapbox.logger
|
|
98
120
|
if options[:log_fetcher]
|
|
99
121
|
type = options[:log_fetcher][:type]
|
|
100
|
-
@log_fetcher = LogFetcher.new(type, options[:log_fetcher])
|
|
122
|
+
@log_fetcher = LogFetcher.new(type, **options[:log_fetcher])
|
|
101
123
|
end
|
|
102
124
|
end
|
|
103
125
|
|
|
104
126
|
class Parameter
|
|
105
127
|
attr_reader \
|
|
106
128
|
:environments,
|
|
107
|
-
:task_role_arn,
|
|
108
|
-
:execution_role_arn,
|
|
109
|
-
:cluster,
|
|
110
129
|
:timeout,
|
|
111
|
-
:launch_type,
|
|
112
130
|
:launch_timeout,
|
|
113
131
|
:launch_retry,
|
|
114
132
|
:retry_interval,
|
|
@@ -116,7 +134,7 @@ module Wrapbox
|
|
|
116
134
|
:max_retry_interval,
|
|
117
135
|
:execution_retry
|
|
118
136
|
|
|
119
|
-
def initialize(environments: [],
|
|
137
|
+
def initialize(environments: [], timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
|
|
120
138
|
b = binding
|
|
121
139
|
method(:initialize).parameters.each do |param|
|
|
122
140
|
instance_variable_set("@#{param[1]}", b.local_variable_get(param[1]))
|
|
@@ -128,30 +146,52 @@ module Wrapbox
|
|
|
128
146
|
task_definition = prepare_task_definition(container_definition_overrides)
|
|
129
147
|
parameter = Parameter.new(**parameters)
|
|
130
148
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
149
|
+
envs = parameters[:environments] || []
|
|
150
|
+
envs += [
|
|
151
|
+
{
|
|
152
|
+
name: CLASS_NAME_ENV,
|
|
153
|
+
value: class_name.to_s,
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
name: METHOD_NAME_ENV,
|
|
157
|
+
value: method_name.to_s,
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
name: METHOD_ARGS_ENV,
|
|
161
|
+
value: MultiJson.dump(args),
|
|
162
|
+
},
|
|
163
|
+
]
|
|
164
|
+
|
|
165
|
+
if @instance_manager
|
|
166
|
+
Thread.new { @instance_manager.start_preparing_instances(1) }
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
run_task(task_definition.task_definition_arn, ["bundle", "exec", "rake", "wrapbox:run"], envs, parameter)
|
|
170
|
+
ensure
|
|
171
|
+
@instance_manager&.terminate_all_instances
|
|
136
172
|
end
|
|
137
173
|
|
|
138
174
|
def run_cmd(cmds, container_definition_overrides: {}, ignore_signal: false, **parameters)
|
|
139
175
|
ths = []
|
|
140
176
|
|
|
141
177
|
task_definition = prepare_task_definition(container_definition_overrides)
|
|
178
|
+
parameter = Parameter.new(**parameters)
|
|
142
179
|
|
|
143
180
|
cmds << nil if cmds.empty?
|
|
181
|
+
|
|
182
|
+
if @instance_manager
|
|
183
|
+
Thread.new { @instance_manager.start_preparing_instances(cmds.size) }
|
|
184
|
+
end
|
|
185
|
+
|
|
144
186
|
cmds.each_with_index do |cmd, idx|
|
|
145
187
|
ths << Thread.new(cmd, idx) do |c, i|
|
|
146
188
|
Thread.current[:cmd_index] = i
|
|
147
189
|
envs = (parameters[:environments] || []) + [{name: "WRAPBOX_CMD_INDEX", value: i.to_s}]
|
|
148
|
-
run_task(
|
|
149
|
-
task_definition.task_definition_arn, nil, nil, nil,
|
|
150
|
-
c&.shellsplit,
|
|
151
|
-
Parameter.new(**parameters.merge(environments: envs))
|
|
152
|
-
)
|
|
190
|
+
run_task(task_definition.task_definition_arn, c&.shellsplit, envs, parameter)
|
|
153
191
|
end
|
|
154
192
|
end
|
|
193
|
+
ThreadsWait.all_waits(ths)
|
|
194
|
+
# Raise an error if some threads have an error
|
|
155
195
|
ths.each(&:join)
|
|
156
196
|
|
|
157
197
|
true
|
|
@@ -172,6 +212,8 @@ module Wrapbox
|
|
|
172
212
|
end
|
|
173
213
|
end
|
|
174
214
|
nil
|
|
215
|
+
ensure
|
|
216
|
+
@instance_manager&.terminate_all_instances
|
|
175
217
|
end
|
|
176
218
|
|
|
177
219
|
private
|
|
@@ -180,24 +222,24 @@ module Wrapbox
|
|
|
180
222
|
!!@task_definition_info
|
|
181
223
|
end
|
|
182
224
|
|
|
183
|
-
def run_task(task_definition_arn,
|
|
184
|
-
cl = parameter.cluster || self.cluster
|
|
225
|
+
def run_task(task_definition_arn, command, environments, parameter)
|
|
185
226
|
execution_try_count = 0
|
|
186
227
|
|
|
228
|
+
ec2_instance_id = @instance_manager&.pop_ec2_instance_id
|
|
187
229
|
begin
|
|
188
|
-
task = create_task(task_definition_arn,
|
|
230
|
+
task = create_task(task_definition_arn, command, environments, parameter, ec2_instance_id)
|
|
189
231
|
return unless task # only Task creation aborted by SignalException
|
|
190
232
|
|
|
191
|
-
@logger.
|
|
233
|
+
@logger.info("#{log_prefix}Launch Task: #{task.task_arn}")
|
|
192
234
|
|
|
193
|
-
wait_task_stopped(
|
|
235
|
+
wait_task_stopped(task.task_arn, parameter.timeout)
|
|
194
236
|
|
|
195
|
-
@logger.
|
|
237
|
+
@logger.info("#{log_prefix}Stop Task: #{task.task_arn}")
|
|
196
238
|
|
|
197
239
|
# Avoid container exit code fetch miss
|
|
198
240
|
sleep WAIT_DELAY
|
|
199
241
|
|
|
200
|
-
task_status = fetch_task_status(
|
|
242
|
+
task_status = fetch_task_status(task.task_arn)
|
|
201
243
|
|
|
202
244
|
# If exit_code is nil, Container is force killed or ECS failed to launch Container by Irregular situation
|
|
203
245
|
error_message = build_error_message(task_definition_name, task.task_arn, task_status)
|
|
@@ -212,18 +254,18 @@ module Wrapbox
|
|
|
212
254
|
raise
|
|
213
255
|
else
|
|
214
256
|
execution_try_count += 1
|
|
215
|
-
@logger.
|
|
257
|
+
@logger.warn("#{log_prefix}Retry Execution after #{EXECUTION_RETRY_INTERVAL} sec (#{execution_try_count}/#{parameter.execution_retry})")
|
|
216
258
|
sleep EXECUTION_RETRY_INTERVAL
|
|
217
259
|
retry
|
|
218
260
|
end
|
|
219
261
|
rescue SignalException
|
|
220
262
|
client.stop_task(
|
|
221
|
-
cluster:
|
|
263
|
+
cluster: @cluster,
|
|
222
264
|
task: task.task_arn,
|
|
223
265
|
reason: "signal interrupted"
|
|
224
266
|
)
|
|
225
|
-
wait_task_stopped(
|
|
226
|
-
@logger.debug("Stop Task: #{task.task_arn}")
|
|
267
|
+
wait_task_stopped(task.task_arn, TERM_TIMEOUT)
|
|
268
|
+
@logger.debug("#{log_prefix}Stop Task: #{task.task_arn}")
|
|
227
269
|
ensure
|
|
228
270
|
if @log_fetcher
|
|
229
271
|
begin
|
|
@@ -232,29 +274,43 @@ module Wrapbox
|
|
|
232
274
|
@logger.warn(e)
|
|
233
275
|
end
|
|
234
276
|
end
|
|
277
|
+
@instance_manager.terminate_instance(ec2_instance_id) if ec2_instance_id
|
|
235
278
|
end
|
|
236
279
|
end
|
|
237
280
|
|
|
238
|
-
def create_task(task_definition_arn,
|
|
239
|
-
cl = parameter.cluster || self.cluster
|
|
240
|
-
launch_type = parameter.launch_type || self.launch_type
|
|
281
|
+
def create_task(task_definition_arn, command, environments, parameter, ec2_instance_id)
|
|
241
282
|
args = Array(args)
|
|
242
283
|
|
|
243
284
|
launch_try_count = 0
|
|
244
285
|
current_retry_interval = parameter.retry_interval
|
|
245
286
|
|
|
246
287
|
begin
|
|
247
|
-
run_task_options = build_run_task_options(task_definition_arn,
|
|
248
|
-
@logger.debug("Task Options: #{run_task_options}")
|
|
249
|
-
|
|
288
|
+
run_task_options = build_run_task_options(task_definition_arn, command, environments, ec2_instance_id)
|
|
289
|
+
@logger.debug("#{log_prefix}Task Options: #{run_task_options}")
|
|
290
|
+
|
|
291
|
+
begin
|
|
292
|
+
resp = client.run_task(run_task_options)
|
|
293
|
+
rescue Aws::ECS::Errors::ThrottlingException
|
|
294
|
+
@logger.warn("#{log_prefix}Failure: Rate exceeded.")
|
|
295
|
+
raise LaunchFailure
|
|
296
|
+
rescue Aws::ECS::Errors::InvalidParameterException => e
|
|
297
|
+
# ec2:DescribeSecurityGroups is called in ecs:RunTask when awsvpc mode is used,
|
|
298
|
+
# and some errors like "Request limit exceeded", "InternalError" etc. caused by
|
|
299
|
+
# ec2:DescribeSecurityGroups are retriable.
|
|
300
|
+
# cf. https://github.com/reproio/wrapbox/issues/32, https://github.com/reproio/wrapbox/issues/43
|
|
301
|
+
raise if !e.message.include?("Request limit exceeded") && !e.message.include?("InternalError")
|
|
302
|
+
|
|
303
|
+
@logger.warn("#{log_prefix}Failure: #{e.message}")
|
|
304
|
+
raise LaunchFailure
|
|
305
|
+
end
|
|
250
306
|
task = resp.tasks[0]
|
|
251
307
|
|
|
252
308
|
resp.failures.each do |failure|
|
|
253
|
-
@logger.
|
|
309
|
+
@logger.warn("#{log_prefix}Failure: Arn=#{failure.arn}, Reason=#{failure.reason}")
|
|
254
310
|
end
|
|
255
311
|
raise LackResource unless task # this case is almost lack of container resource.
|
|
256
312
|
|
|
257
|
-
@logger.debug("Create Task: #{task.task_arn}")
|
|
313
|
+
@logger.debug("#{log_prefix}Create Task: #{task.task_arn}")
|
|
258
314
|
|
|
259
315
|
@log_fetcher.run(task: task) if @log_fetcher
|
|
260
316
|
|
|
@@ -262,17 +318,17 @@ module Wrapbox
|
|
|
262
318
|
sleep WAIT_DELAY
|
|
263
319
|
|
|
264
320
|
begin
|
|
265
|
-
wait_task_running(
|
|
321
|
+
wait_task_running(task.task_arn, parameter.launch_timeout)
|
|
266
322
|
task
|
|
267
|
-
rescue
|
|
323
|
+
rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout
|
|
268
324
|
client.stop_task(
|
|
269
|
-
cluster:
|
|
325
|
+
cluster: @cluster,
|
|
270
326
|
task: task.task_arn,
|
|
271
327
|
reason: "launch timeout"
|
|
272
328
|
)
|
|
273
329
|
raise
|
|
274
|
-
rescue
|
|
275
|
-
task_status = fetch_task_status(
|
|
330
|
+
rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitFailure
|
|
331
|
+
task_status = fetch_task_status(task.task_arn)
|
|
276
332
|
|
|
277
333
|
case task_status[:last_status]
|
|
278
334
|
when "RUNNING"
|
|
@@ -288,71 +344,62 @@ module Wrapbox
|
|
|
288
344
|
end
|
|
289
345
|
end
|
|
290
346
|
rescue LackResource
|
|
291
|
-
@logger.
|
|
292
|
-
put_waiting_task_count_metric
|
|
347
|
+
@logger.warn("#{log_prefix}Failed to create task, because of lack resource")
|
|
348
|
+
put_waiting_task_count_metric
|
|
293
349
|
|
|
294
350
|
if launch_try_count >= parameter.launch_retry
|
|
295
351
|
raise
|
|
296
352
|
else
|
|
297
353
|
launch_try_count += 1
|
|
298
|
-
|
|
299
|
-
|
|
354
|
+
retry_interval = current_retry_interval/2 + rand(current_retry_interval/2)
|
|
355
|
+
@logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
|
|
356
|
+
sleep retry_interval
|
|
300
357
|
current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
|
|
301
358
|
retry
|
|
302
359
|
end
|
|
303
360
|
rescue LaunchFailure
|
|
304
361
|
if launch_try_count >= parameter.launch_retry
|
|
305
|
-
task_status = fetch_task_status(
|
|
362
|
+
task_status = fetch_task_status(task.task_arn)
|
|
306
363
|
raise LaunchFailure, build_error_message(task_definition_name, task.task_arn, task_status)
|
|
307
364
|
else
|
|
308
365
|
launch_try_count += 1
|
|
309
|
-
|
|
310
|
-
|
|
366
|
+
retry_interval = current_retry_interval/2 + rand(current_retry_interval/2)
|
|
367
|
+
@logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
|
|
368
|
+
sleep retry_interval
|
|
311
369
|
current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
|
|
312
370
|
retry
|
|
313
371
|
end
|
|
314
372
|
rescue SignalException
|
|
315
373
|
if task
|
|
316
374
|
client.stop_task(
|
|
317
|
-
cluster:
|
|
375
|
+
cluster: @cluster,
|
|
318
376
|
task: task.task_arn,
|
|
319
377
|
reason: "signal interrupted"
|
|
320
378
|
)
|
|
321
|
-
wait_task_stopped(
|
|
322
|
-
@logger.debug("Stop Task: #{task.task_arn}")
|
|
379
|
+
wait_task_stopped(task.task_arn, TERM_TIMEOUT)
|
|
380
|
+
@logger.debug("#{log_prefix}Stop Task: #{task.task_arn}")
|
|
323
381
|
nil
|
|
324
382
|
end
|
|
325
383
|
end
|
|
326
384
|
end
|
|
327
385
|
|
|
328
|
-
def
|
|
329
|
-
|
|
330
|
-
if timeout
|
|
331
|
-
w.delay = WAIT_DELAY
|
|
332
|
-
w.max_attempts = timeout / w.delay
|
|
333
|
-
else
|
|
334
|
-
w.max_attempts = nil
|
|
335
|
-
end
|
|
336
|
-
end
|
|
386
|
+
def wait_task_running(task_arn, launch_timeout)
|
|
387
|
+
@task_waiter.wait_task_running(task_arn, timeout: launch_timeout)
|
|
337
388
|
end
|
|
338
389
|
|
|
339
|
-
def
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
def wait_task_stopped(cluster, task_arn, execution_timeout)
|
|
344
|
-
wait_until_with_timeout(cluster, task_arn, execution_timeout, :tasks_stopped)
|
|
345
|
-
rescue Aws::Waiters::Errors::TooManyAttemptsError
|
|
390
|
+
def wait_task_stopped(task_arn, execution_timeout)
|
|
391
|
+
@task_waiter.wait_task_stopped(task_arn, timeout: execution_timeout)
|
|
392
|
+
rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout
|
|
346
393
|
client.stop_task({
|
|
347
|
-
cluster: cluster,
|
|
394
|
+
cluster: @cluster,
|
|
348
395
|
task: task_arn,
|
|
349
396
|
reason: "process timeout",
|
|
350
397
|
})
|
|
351
398
|
raise ExecutionTimeout, "Task #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{execution_timeout}"
|
|
352
399
|
end
|
|
353
400
|
|
|
354
|
-
def fetch_task_status(
|
|
355
|
-
task = client.describe_tasks(cluster: cluster, tasks: [task_arn]).tasks[0]
|
|
401
|
+
def fetch_task_status(task_arn)
|
|
402
|
+
task = client.describe_tasks(cluster: @cluster, tasks: [task_arn]).tasks[0]
|
|
356
403
|
container = task.containers.find { |c| c.name == main_container_name }
|
|
357
404
|
{
|
|
358
405
|
last_status: task.last_status,
|
|
@@ -385,7 +432,7 @@ module Wrapbox
|
|
|
385
432
|
end
|
|
386
433
|
end
|
|
387
434
|
|
|
388
|
-
@logger.debug("Container Definitions: #{overrided_container_definitions}")
|
|
435
|
+
@logger.debug("#{log_prefix}Container Definitions: #{overrided_container_definitions}")
|
|
389
436
|
register_retry_count = 0
|
|
390
437
|
begin
|
|
391
438
|
client.register_task_definition({
|
|
@@ -396,10 +443,11 @@ module Wrapbox
|
|
|
396
443
|
container_definitions: overrided_container_definitions,
|
|
397
444
|
volumes: volumes,
|
|
398
445
|
requires_compatibilities: requires_compatibilities,
|
|
399
|
-
task_role_arn: task_role_arn,
|
|
400
|
-
execution_role_arn: execution_role_arn
|
|
446
|
+
task_role_arn: @task_role_arn,
|
|
447
|
+
execution_role_arn: @execution_role_arn,
|
|
448
|
+
tags: tags,
|
|
401
449
|
}).task_definition
|
|
402
|
-
rescue Aws::ECS::Errors::ClientException
|
|
450
|
+
rescue Aws::ECS::Errors::ClientException, Aws::ECS::Errors::ThrottlingException
|
|
403
451
|
raise if register_retry_count > 2
|
|
404
452
|
register_retry_count += 1
|
|
405
453
|
sleep 2
|
|
@@ -407,6 +455,14 @@ module Wrapbox
|
|
|
407
455
|
end
|
|
408
456
|
end
|
|
409
457
|
|
|
458
|
+
def cmd_index
|
|
459
|
+
Thread.current[:cmd_index]
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
def log_prefix
|
|
463
|
+
cmd_index ? "##{cmd_index} " : ""
|
|
464
|
+
end
|
|
465
|
+
|
|
410
466
|
def client
|
|
411
467
|
return @client if @client
|
|
412
468
|
|
|
@@ -423,7 +479,7 @@ module Wrapbox
|
|
|
423
479
|
@cloud_watch_client = Aws::CloudWatch::Client.new(options)
|
|
424
480
|
end
|
|
425
481
|
|
|
426
|
-
def put_waiting_task_count_metric
|
|
482
|
+
def put_waiting_task_count_metric
|
|
427
483
|
cloud_watch_client.put_metric_data(
|
|
428
484
|
namespace: "wrapbox",
|
|
429
485
|
metric_data: [
|
|
@@ -431,7 +487,7 @@ module Wrapbox
|
|
|
431
487
|
dimensions: [
|
|
432
488
|
{
|
|
433
489
|
name: "ClusterName",
|
|
434
|
-
value: cluster
|
|
490
|
+
value: @cluster,
|
|
435
491
|
},
|
|
436
492
|
],
|
|
437
493
|
timestamp: Time.now,
|
|
@@ -441,54 +497,53 @@ module Wrapbox
|
|
|
441
497
|
)
|
|
442
498
|
end
|
|
443
499
|
|
|
444
|
-
def build_run_task_options(task_definition_arn,
|
|
445
|
-
env = environments
|
|
446
|
-
env += [
|
|
447
|
-
{
|
|
448
|
-
name: CLASS_NAME_ENV,
|
|
449
|
-
value: class_name.to_s,
|
|
450
|
-
},
|
|
451
|
-
{
|
|
452
|
-
name: METHOD_NAME_ENV,
|
|
453
|
-
value: method_name.to_s,
|
|
454
|
-
},
|
|
455
|
-
{
|
|
456
|
-
name: METHOD_ARGS_ENV,
|
|
457
|
-
value: MultiJson.dump(args),
|
|
458
|
-
},
|
|
459
|
-
] if class_name && method_name && args
|
|
500
|
+
def build_run_task_options(task_definition_arn, command, environments, ec2_instance_id)
|
|
460
501
|
overrides = {
|
|
461
502
|
container_overrides: [
|
|
462
503
|
{
|
|
463
504
|
name: main_container_name,
|
|
464
|
-
environment:
|
|
505
|
+
environment: environments,
|
|
465
506
|
}.tap { |o| o[:command] = command if command },
|
|
466
507
|
*container_definitions.drop(1).map do |c|
|
|
467
508
|
{
|
|
468
509
|
name: c[:name],
|
|
469
|
-
environment:
|
|
510
|
+
environment: environments,
|
|
470
511
|
}
|
|
471
512
|
end
|
|
472
513
|
],
|
|
473
514
|
}
|
|
474
|
-
|
|
475
|
-
overrides[:task_role_arn] = role_arn if role_arn
|
|
515
|
+
overrides[:task_role_arn] = @task_role_arn if @task_role_arn
|
|
476
516
|
|
|
477
|
-
|
|
478
|
-
|
|
517
|
+
additional_placement_constraints = []
|
|
518
|
+
if ec2_instance_id
|
|
519
|
+
additional_placement_constraints << { type: "memberOf", expression: "ec2InstanceId == #{ec2_instance_id}" }
|
|
520
|
+
end
|
|
521
|
+
options = {
|
|
522
|
+
cluster: @cluster,
|
|
523
|
+
enable_execute_command: enable_execute_command,
|
|
479
524
|
task_definition: task_definition_arn,
|
|
480
525
|
overrides: overrides,
|
|
481
526
|
placement_strategy: placement_strategy,
|
|
482
|
-
placement_constraints: placement_constraints,
|
|
483
|
-
launch_type: launch_type,
|
|
527
|
+
placement_constraints: placement_constraints + additional_placement_constraints,
|
|
484
528
|
network_configuration: network_configuration,
|
|
485
529
|
started_by: "wrapbox-#{Wrapbox::VERSION}",
|
|
530
|
+
enable_ecs_managed_tags: enable_ecs_managed_tags,
|
|
531
|
+
propagate_tags: propagate_tags,
|
|
486
532
|
}
|
|
533
|
+
if @capacity_provider_strategy.empty?
|
|
534
|
+
options[:launch_type] = @launch_type if @launch_type
|
|
535
|
+
else
|
|
536
|
+
if @launch_type
|
|
537
|
+
@logger.warn("#{log_prefix}Ignore --launch_type and launch_type in the configuration file when specified capacity_provider_strategy in the configuration file")
|
|
538
|
+
end
|
|
539
|
+
options[:capacity_provider_strategy] = @capacity_provider_strategy
|
|
540
|
+
end
|
|
541
|
+
options
|
|
487
542
|
end
|
|
488
543
|
|
|
489
544
|
def build_error_message(task_definition_name, task_arn, task_status)
|
|
490
545
|
error_message = "Task #{task_definition_name} is failed. task=#{task_arn}, "
|
|
491
|
-
error_message << "cmd_index=#{
|
|
546
|
+
error_message << "cmd_index=#{cmd_index}, " if cmd_index
|
|
492
547
|
error_message << "exit_code=#{task_status[:exit_code]}, task_stopped_reason=#{task_status[:stopped_reason]}, container_stopped_reason=#{task_status[:container_stopped_reason]}"
|
|
493
548
|
error_message
|
|
494
549
|
end
|
|
@@ -502,21 +557,30 @@ module Wrapbox
|
|
|
502
557
|
method_option :cluster, aliases: "-c"
|
|
503
558
|
method_option :cpu, type: :numeric
|
|
504
559
|
method_option :memory, type: :numeric
|
|
560
|
+
method_option :working_directory, aliases: "-w", type: :string
|
|
505
561
|
method_option :environments, aliases: "-e"
|
|
506
562
|
method_option :task_role_arn
|
|
507
563
|
method_option :timeout, type: :numeric
|
|
508
|
-
method_option :launch_type, type: :string,
|
|
564
|
+
method_option :launch_type, type: :string, enum: ["EC2", "FARGATE"]
|
|
509
565
|
method_option :launch_timeout, type: :numeric
|
|
510
566
|
method_option :launch_retry, type: :numeric
|
|
511
567
|
method_option :execution_retry, type: :numeric
|
|
512
568
|
method_option :max_retry_interval, type: :numeric
|
|
513
569
|
method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), ECS Tasks continue running"
|
|
570
|
+
method_option :tags, type: :string, aliases: "-t", repeatable: true
|
|
571
|
+
method_option :propagate_tags, type: :string, enum: ["TASK_DEFINITION", "SERVICE"]
|
|
572
|
+
method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Verbose mode"
|
|
514
573
|
def run_cmd(*args)
|
|
574
|
+
Wrapbox.logger.level = :debug if options[:verbose]
|
|
515
575
|
Wrapbox.load_config(options[:config])
|
|
516
576
|
config = Wrapbox.configs[options[:config_name]]
|
|
517
577
|
environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
|
|
518
578
|
{name: k, value: v}
|
|
519
579
|
end
|
|
580
|
+
tags = options.fetch(:tags, []).map do |kv|
|
|
581
|
+
k, v = kv.split("=", 2)
|
|
582
|
+
{key: k, value: v}
|
|
583
|
+
end.presence
|
|
520
584
|
run_options = {
|
|
521
585
|
cluster: options[:cluster],
|
|
522
586
|
task_role_arn: options[:task_role_arn],
|
|
@@ -527,9 +591,11 @@ module Wrapbox
|
|
|
527
591
|
execution_retry: options[:execution_retry],
|
|
528
592
|
max_retry_interval: options[:max_retry_interval],
|
|
529
593
|
ignore_signal: options[:ignore_signal],
|
|
594
|
+
tags: tags,
|
|
595
|
+
propagate_tags: options[:propagate_tags],
|
|
530
596
|
}.reject { |_, v| v.nil? }
|
|
531
|
-
if options[:cpu] || options[:memory]
|
|
532
|
-
container_definition_overrides = {cpu: options[:cpu], memory: options[:memory]}.reject { |_, v| v.nil? }
|
|
597
|
+
if options[:cpu] || options[:memory] || options[:working_directory]
|
|
598
|
+
container_definition_overrides = {cpu: options[:cpu], memory: options[:memory], working_directory: options[:working_directory]}.reject { |_, v| v.nil? }
|
|
533
599
|
else
|
|
534
600
|
container_definition_overrides = {}
|
|
535
601
|
end
|
data/lib/wrapbox/version.rb
CHANGED
data/lib/wrapbox.rb
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
require "logger"
|
|
2
|
+
|
|
1
3
|
module Wrapbox
|
|
2
4
|
CLASS_NAME_ENV = "WRAPBOX_CLASS_NAME".freeze
|
|
3
5
|
METHOD_NAME_ENV = "WRAPBOX_METHOD_NAME".freeze
|
|
4
6
|
METHOD_ARGS_ENV = "WRAPBOX_METHOD_ARGS".freeze
|
|
5
7
|
|
|
6
8
|
class << self
|
|
9
|
+
attr_accessor :logger
|
|
10
|
+
|
|
7
11
|
def load_config(filename)
|
|
8
12
|
configs.load_yaml(filename)
|
|
9
13
|
end
|
|
@@ -16,16 +20,25 @@ module Wrapbox
|
|
|
16
20
|
yield configs
|
|
17
21
|
end
|
|
18
22
|
|
|
19
|
-
def run(*args,
|
|
20
|
-
|
|
21
|
-
|
|
23
|
+
def run(*args, config_name: nil, **options)
|
|
24
|
+
get_config(config_name).run(*args, **options)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def run_cmd(*args, config_name: nil, **options)
|
|
28
|
+
get_config(config_name).run_cmd(*args, **options)
|
|
22
29
|
end
|
|
23
30
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def get_config(config_name)
|
|
34
|
+
@configs.get(config_name) or
|
|
35
|
+
raise RuntimeError, %Q{The configuration "#{config_name}" is not registered}
|
|
27
36
|
end
|
|
28
37
|
end
|
|
38
|
+
|
|
39
|
+
$stdout.sync = true
|
|
40
|
+
self.logger = Logger.new($stdout)
|
|
41
|
+
self.logger.level = :info
|
|
29
42
|
end
|
|
30
43
|
|
|
31
44
|
require "wrapbox/version"
|
data/wrapbox.gemspec
CHANGED
|
@@ -20,15 +20,17 @@ Gem::Specification.new do |spec|
|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
21
21
|
spec.require_paths = ["lib"]
|
|
22
22
|
|
|
23
|
+
spec.add_runtime_dependency "aws-sdk-ec2", "~> 1"
|
|
23
24
|
spec.add_runtime_dependency "aws-sdk-ecs", "~> 1"
|
|
24
25
|
spec.add_runtime_dependency "aws-sdk-cloudwatch", "~> 1"
|
|
25
26
|
spec.add_runtime_dependency "activesupport", ">= 4"
|
|
26
27
|
spec.add_runtime_dependency "docker-api"
|
|
27
28
|
spec.add_runtime_dependency "multi_json"
|
|
28
|
-
spec.add_runtime_dependency "thor"
|
|
29
|
+
spec.add_runtime_dependency "thor", ">= 1"
|
|
30
|
+
spec.add_runtime_dependency "thwait"
|
|
29
31
|
|
|
30
|
-
spec.add_development_dependency "bundler"
|
|
31
|
-
spec.add_development_dependency "rake"
|
|
32
|
+
spec.add_development_dependency "bundler"
|
|
33
|
+
spec.add_development_dependency "rake"
|
|
32
34
|
spec.add_development_dependency "rspec", "~> 3.0"
|
|
33
35
|
spec.add_development_dependency "webmock"
|
|
34
36
|
spec.add_development_dependency "tapp"
|