wrapbox 0.9.0 → 0.10.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 00df8d3c0e46ae38cdef3ada6dde9db32adf2313b03d910591e98a9129b1d787
4
- data.tar.gz: 61739c6ac9544da2b2ea83fd538831e6782ee78d9e0d2ddc8f2424e409d48474
3
+ metadata.gz: bdb5f8dc31647c10b941f003cc2c0c16127210e38e922abe1ed6f24a691ac497
4
+ data.tar.gz: 8fce00a8ada4301f82a079c57a51a9d85fb4502ae95b13ce0a0582d83f3eef70
5
5
  SHA512:
6
- metadata.gz: 97c9a51c95c80f93ec16443eaca2e91890da6a1d7574f6d62be200ac2db8a997bcae61dc74da0467b3aa4aac751c81db86ea3943f86988ff78644b52efad92d3
7
- data.tar.gz: bae99aa04b2540582132f769d0db1b20dd171956cef2bd8fb08b4d16f430365c56ba86a0c706ad37b6499ee0b037093828703d752ae0672f9d16e4ad0a1ace59
6
+ metadata.gz: d49adcac17d1390776a19992e9636680617dda700f6084c8e6b06e17d300b133cf452ed7f80d50eea84c2e0b6b9e97121629b4e3255263726172e520d3d7dfe2
7
+ data.tar.gz: 3ef0163a2dfd7180f8dfbff4508f4c7171f7bfd29a828c71744523e1bcdc647553cb37aa9a353ab5c06a796224168a08c362662b5211ea770b062cd2eb3a43ad
data/.dockerignore CHANGED
@@ -7,6 +7,5 @@ doc
7
7
  pkg
8
8
  spec/reports
9
9
  tmp
10
- .travis.yml
11
10
 
12
11
  .envrc
@@ -0,0 +1,30 @@
1
+ name: Testing on Ubuntu
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ timeout-minutes: 10
9
+ strategy:
10
+ fail-fast: false
11
+ matrix:
12
+ ruby:
13
+ - 2.7
14
+ - 3.0
15
+ - 3.1
16
+ - 3.2
17
+ os:
18
+ - ubuntu-latest
19
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
20
+ steps:
21
+ - uses: actions/checkout@v2
22
+ - uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby }}
25
+ - name: unit testing
26
+ env:
27
+ CI: true
28
+ run: |
29
+ bundle install --jobs 4 --retry 3
30
+ bundle exec rake spec
data/README.md CHANGED
@@ -78,6 +78,13 @@ Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :docke
78
78
  Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development"}])
79
79
  ```
80
80
 
81
+ If ECS runner cannot create task, it puts custom metric data to CloudWatch.
82
+ Custom metric data is `wrapbox/WaitingTaskCount` that has `ClusterName` dimension.
83
+ And, it retry launching until retry count reach `launch_retry`.
84
+
85
+ After task exited, Wrapbox checks main container exit code.
86
+ If exit code is not 0, Wrapbox raise error.
87
+
81
88
  ## Config
82
89
 
83
90
  ### Common
@@ -88,20 +95,25 @@ Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development
88
95
 
89
96
  ### for ECS
90
97
 
91
- | name | desc |
92
- | -------------------- | ------------------------------------------------ |
93
- | cluster | target ECS cluster name |
94
- | region | region of ECS cluster |
95
- | container_definitions | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
96
- | task_role_arn | see. http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html |
97
- | volumes | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
98
- | placement_constraints | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
99
- | placement_strategy | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
100
- | launch_type | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
101
- | newtork_mode | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
102
- | network_configuration | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
103
- | cpu | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
104
- | memory | see. http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
98
+ | name | desc |
99
+ | -------------------------- | ------------------------------------------------ |
100
+ | cluster | target ECS cluster name |
101
+ | region | region of ECS cluster |
102
+ | container_definitions | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
103
+ | task_role_arn | see http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html |
104
+ | volumes | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
105
+ | placement_constraints | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
106
+ | placement_strategy | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
107
+ | launch_type | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
108
+ | network_mode | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
109
+ | network_configuration | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
110
+ | capacity_provider_strategy | see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/ECS/Client.html#run_task-instance_method |
111
+ | cpu | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
112
+ | memory | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
113
+ | enable_ecs_managed_tags | see https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
114
+ | tags | tags of task definitions. see also https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method |
115
+ | propagate_tags | specify `"TASK_DEFINITION"` if you want to propagate tags to tasks. see also https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method |
116
+ | launch_instances | specify `launch_template` (required), `instance_type`, and `tag_specifications` for [Aws::EC2::Client#run_instances](https://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Client.html#run_instances-instance_method). You can also specify `wait_until_instance_terminated` (default: true) |
105
117
 
106
118
  `WRAPBOX_CMD_INDEX` environment variable is available in `run_cmd` and you can distinguish logs from each command like below:
107
119
 
@@ -119,28 +131,59 @@ log_configuration:
119
131
  | -------------------- | ----------------------------------------------------------- |
120
132
  | container_definitions | only use `image`, `cpu`, `memory`, and `memory_reservation` |
121
133
  | keep_container | If true, doesn't delete the container when the command ends |
122
- | use_sudo | If true, invoke `sudo docker` command |
123
134
 
124
135
  ## API
125
136
 
126
- #### `run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)`
137
+ ### `Wrapbox.run`
127
138
 
128
- #### `run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)`
129
-
130
- following options is only for ECS.
131
-
132
- - task_role_arn
133
- - cluster
134
- - timeout
135
- - launch_timeout
136
- - launch_retry
139
+ ```ruby
140
+ Wrapbox.run(class_name, method_name, args,
141
+ runner: nil, # The "runner" value is used in the configuration if it is nil.
142
+ config_name: nil, # "default" configuration is used if it is nil.
143
+ cluster: nil, # Available only for ECS runner. The "cluster" value in the configuration is used if it is nil.
144
+ launch_type: "EC2", # Available only for ECS runner. The "launch_type" value in the configuration is used if it is nil.
145
+ task_role_arn: nil, # Available only for ECS runner. The "task_role_arn" value in the configuration is used if it is nil.
146
+ execution_role_arn: nil, # Available only for ECS runner. The "execution_role_arn" value in the configuration is used if it is nil.
147
+ tags: nil, # Available only for ECS runner. The "tags" value in the configuration is used if it is nil.
148
+ propagate_tags: nil, # Available only for ECS runner. The "propagate_tags" value in the configuration is used if it is nil.
149
+ container_definition_overrides: {},
150
+ environments: [],
151
+ timeout: 3600 * 24, # Available only for ECS runner. # Available only for ECS runner.
152
+ launch_timeout: 60 * 10, # Available only for ECS runner.
153
+ launch_retry: 10, # Available only for ECS runner.
154
+ retry_interval: 1, # Available only for ECS runner.
155
+ retry_interval_multiplier: 2, # Available only for ECS runner.
156
+ max_retry_interval: 120, # Available only for ECS runner.
157
+ execution_retry: 0, # Available only for ECS runner.
158
+ keep_container: nil, # Available only for Docker runner. The "keep_container" value in the configuration is used if it is nil.
159
+ )
160
+ ```
137
161
 
138
- If Wrapbox cannot create task, it puts custom metric data to CloudWatch.
139
- Custom metric data is `wrapbox/WaitingTaskCount` that has `ClusterName` dimension.
140
- And, it retry launching until retry count reach `launch_retry`.
162
+ ### `Wrapbox.run_cmd`
141
163
 
142
- After task exited, Wrapbox checks main container exit code.
143
- If exit code is not 0, Wrapbox raise error.
164
+ ```ruby
165
+ Wrapbox.run_cmd(*cmd,
166
+ runner: nil, # The "runner" value is used in the configuration if it is nil.
167
+ config_name: nil, # "default" configuration is used if it is nil.
168
+ cluster: nil, # Available only for ECS runner. The "cluster" value in the configuration is used if it is nil.
169
+ launch_type: "EC2", # Available only for ECS runner. The "launch_type" value in the configuration is used if it is nil.
170
+ task_role_arn: nil, # Available only for ECS runner. The "task_role_arn" value in the configuration is used if it is nil.
171
+ execution_role_arn: nil, # Available only for ECS runner. The "execution_role_arn" value in the configuration is used if it is nil.
172
+ tags: nil, # Available only for ECS runner. The "tags" value in the configuration is used if it is nil.
173
+ propagate_tags: nil, # Available only for ECS runner. The "propagate_tags" value in the configuration is used if it is nil.
174
+ container_definition_overrides: {},
175
+ ignore_signal: false,
176
+ environments: [],
177
+ timeout: 3600 * 24, # Available only for ECS runner. # Available only for ECS runner.
178
+ launch_timeout: 60 * 10, # Available only for ECS runner.
179
+ launch_retry: 10, # Available only for ECS runner.
180
+ retry_interval: 1, # Available only for ECS runner.
181
+ retry_interval_multiplier: 2, # Available only for ECS runner.
182
+ max_retry_interval: 120, # Available only for ECS runner.
183
+ execution_retry: 0, # Available only for ECS runner.
184
+ keep_container: nil, # Available only for Docker runner. The "keep_container" value in the configuration is used if it is nil.
185
+ )
186
+ ```
144
187
 
145
188
  ## Development
146
189
 
@@ -148,6 +191,27 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
148
191
 
149
192
  To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
150
193
 
194
+ ### How to test
195
+
196
+ The following environment variables are required to run all tests.
197
+
198
+ Name | Description
199
+ -----|----------------
200
+ RUN_AWS_SPECS | Set "true" to run tests with `aws` set to true. You should also set credentials for AWS account to run ECS tasks.
201
+ ECS_CLUSTER | A cluster used in tests. "default" cluster is used if this variable is not set.
202
+ OVERRIDDEN_ECS_CLUSTER | A cluster used in tests that ensure `cluster` parameter.
203
+ LAUNCH_TEMPLATE_ID | A launch template used in tests that ensure `launch_instances` configuration.
204
+
205
+
206
+ ```
207
+ env \
208
+ RUN_AWS_SPECS=true \
209
+ ECS_CLUSTER='some_cluster' \
210
+ OVERRIDDEN_ECS_CLUSTER='another_cluster' \
211
+ LAUNCH_TEMPLATE_ID=lt-xxxxxxxxxxxxxxxxx \
212
+ bundle exec rspec
213
+ ```
214
+
151
215
  ## Contributing
152
216
 
153
217
  Bug reports and pull requests are welcome on GitHub at https://github.com/reproio/wrapbox.
@@ -8,7 +8,12 @@ module Wrapbox
8
8
  end
9
9
 
10
10
  def load_yaml(yaml_file)
11
- configs = YAML.load(ERB.new(File.read(yaml_file)).result)
11
+ file = ERB.new(File.read(yaml_file)).result
12
+ configs = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new("4.0.0")
13
+ YAML.load(file, aliases: true)
14
+ else
15
+ YAML.load(file)
16
+ end
12
17
  configs.each do |name, configuration|
13
18
  load_config(name, configuration.merge("name" => name))
14
19
  end
@@ -1,3 +1,4 @@
1
+ require "active_support"
1
2
  require "active_support/core_ext/hash"
2
3
  require "active_support/core_ext/string"
3
4
 
@@ -17,6 +18,7 @@ module Wrapbox
17
18
  :volumes,
18
19
  :placement_constraints,
19
20
  :placement_strategy,
21
+ :capacity_provider_strategy,
20
22
  :launch_type,
21
23
  :requires_compatibilities,
22
24
  :task_definition,
@@ -28,7 +30,12 @@ module Wrapbox
28
30
  :task_role_arn,
29
31
  :execution_role_arn,
30
32
  :keep_container,
31
- :log_fetcher
33
+ :log_fetcher,
34
+ :tags,
35
+ :enable_ecs_managed_tags,
36
+ :propagate_tags,
37
+ :launch_instances,
38
+ :enable_execute_command,
32
39
  ) do
33
40
  def self.load_config(config)
34
41
  new(
@@ -46,6 +53,7 @@ module Wrapbox
46
53
  config["volumes"]&.map(&:deep_symbolize_keys) || [],
47
54
  config["placement_constraints"]&.map(&:deep_symbolize_keys) || [],
48
55
  config["placement_strategy"]&.map(&:deep_symbolize_keys) || [],
56
+ config["capacity_provider_strategy"]&.map(&:deep_symbolize_keys) || [],
49
57
  config["launch_type"],
50
58
  config["requires_compatibilities"] || ["EC2"],
51
59
  config["task_definition"]&.deep_symbolize_keys,
@@ -57,7 +65,12 @@ module Wrapbox
57
65
  config["task_role_arn"],
58
66
  config["execution_role_arn"],
59
67
  config["keep_container"],
60
- config["log_fetcher"]&.deep_symbolize_keys
68
+ config["log_fetcher"]&.deep_symbolize_keys,
69
+ config["tags"],
70
+ config["enable_ecs_managed_tags"],
71
+ config["propagate_tags"],
72
+ config["launch_instances"]&.deep_symbolize_keys,
73
+ config["enable_execute_command"]
61
74
  )
62
75
  end
63
76
 
@@ -67,19 +80,23 @@ module Wrapbox
67
80
  super
68
81
  end
69
82
 
70
- def build_runner(overrided_runner = nil)
83
+ def runner_class(overrided_runner = nil)
71
84
  r = overrided_runner || runner
72
85
  raise "#{r} is unsupported runner" unless AVAILABLE_RUNNERS.include?(r.to_sym)
73
86
  require "wrapbox/runner/#{r}"
74
- Wrapbox::Runner.const_get(r.to_s.camelcase).new(to_h)
87
+ Wrapbox::Runner.const_get(r.to_s.camelcase)
75
88
  end
76
89
 
77
90
  def run(class_name, method_name, args, runner: nil, **options)
78
- build_runner(runner).run(class_name, method_name, args, **options)
91
+ klass = runner_class(runner)
92
+ overridable_options, parameters = klass.split_overridable_options_and_parameters(options)
93
+ klass.new(to_h.merge(overridable_options)).run(class_name, method_name, args, **parameters)
79
94
  end
80
95
 
81
96
  def run_cmd(*cmd, runner: nil, **options)
82
- build_runner(runner).run_cmd(*cmd, **options)
97
+ klass = runner_class(runner)
98
+ overridable_options, parameters = klass.split_overridable_options_and_parameters(options)
99
+ klass.new(to_h.merge(overridable_options)).run_cmd(*cmd, **parameters)
83
100
  end
84
101
  end
85
102
  end
@@ -27,6 +27,14 @@ module Wrapbox
27
27
 
28
28
  def run(task:)
29
29
  @loop_thread = Thread.start do
30
+ # It smees that task.contaienrs is empty
31
+ # if capacity_provider_strategy is specified and there are no remaining capacity
32
+ while task.containers.empty?
33
+ Wrapbox.logger.warn("The task has no containers, so fetch it again")
34
+ sleep 10
35
+ task = ecs_client.describe_tasks(cluster: task.cluster_arn, tasks: [task.task_arn]).tasks.first
36
+ end
37
+
30
38
  main_loop(task)
31
39
  end
32
40
  end
@@ -45,30 +53,34 @@ module Wrapbox
45
53
  log_group_name: @log_group,
46
54
  log_stream_names: log_stream_names,
47
55
  filter_pattern: @filter_pattern,
48
- interleaved: true,
49
56
  }.compact
50
57
  @max_timestamp = ((Time.now.to_f - 120) * 1000).round
51
58
 
52
59
  until @stop do
53
60
  filter_log_opts[:start_time] = @max_timestamp + 1
54
- resp = client.filter_log_events(filter_log_opts) rescue nil
55
- resp&.each do |r|
56
- r.events.each do |ev|
57
- next if @displayed_event_ids.member?(ev.event_id)
58
- display_message(ev)
59
- @displayed_event_ids[ev.event_id] = ev.timestamp
60
- @max_timestamp = ev.timestamp if @max_timestamp < ev.timestamp
61
+ begin
62
+ client.filter_log_events(filter_log_opts).each do |r|
63
+ r.events.each do |ev|
64
+ next if @displayed_event_ids.member?(ev.event_id)
65
+ display_message(ev)
66
+ @displayed_event_ids[ev.event_id] = ev.timestamp
67
+ @max_timestamp = ev.timestamp if @max_timestamp < ev.timestamp
68
+ end
61
69
  end
62
- end
63
- Thread.start do
70
+
64
71
  @displayed_event_ids.each do |event_id, ts|
65
72
  if ts < (Time.now.to_f - 600) * 1000
66
73
  @displayed_event_ids.delete(event_id)
67
74
  end
68
75
  end
69
- end.tap do
70
- sleep @delay
71
- end.join
76
+ rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
77
+ # Ignore the error because it is an error like "The specified log stream does not exist.",
78
+ # which occurs when the log stream hasn't been created yet, that is, the task hasn't started yet.
79
+ rescue Aws::CloudWatchLogs::Errors::ThrottlingException
80
+ Wrapbox.logger.warn("Failed to fetch logs due to Aws::CloudWatchLogs::Errors::ThrottlingException")
81
+ end
82
+
83
+ sleep @delay
72
84
  end
73
85
  end
74
86
 
@@ -4,6 +4,8 @@ require "docker"
4
4
  require "thor"
5
5
  require "shellwords"
6
6
 
7
+ require "wrapbox"
8
+
7
9
  module Wrapbox
8
10
  module Runner
9
11
  class Docker
@@ -14,6 +16,17 @@ module Wrapbox
14
16
  :container_definition,
15
17
  :keep_container
16
18
 
19
+ def self.split_overridable_options_and_parameters(options)
20
+ opts = options.dup
21
+ overridable_options = {}
22
+ %i[keep_container].each do |key|
23
+ value = opts.delete(key)
24
+ overridable_options[key] = value if value
25
+ end
26
+
27
+ [overridable_options, opts]
28
+ end
29
+
17
30
  def initialize(options)
18
31
  @name = options[:name]
19
32
  @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions]
@@ -136,6 +149,7 @@ module Wrapbox
136
149
  method_option :config_name, aliases: "-n", required: true, default: "default"
137
150
  method_option :cpu, type: :numeric
138
151
  method_option :memory, type: :numeric
152
+ method_option :working_directory, aliases: "-w", type: :string
139
153
  method_option :environments, aliases: "-e"
140
154
  method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), Docker container continue running"
141
155
  method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Verbose mode"
@@ -146,8 +160,8 @@ module Wrapbox
146
160
  environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
147
161
  {name: k, value: v}
148
162
  end
149
- if options[:cpu] || options[:memory]
150
- container_definition_overrides = {cpu: options[:cpu], memory: options[:memory]}.reject { |_, v| v.nil? }
163
+ if options[:cpu] || options[:memory] || options[:working_directory]
164
+ container_definition_overrides = {cpu: options[:cpu], memory: options[:memory], working_directory: options[:working_directory]}.reject { |_, v| v.nil? }
151
165
  else
152
166
  container_definition_overrides = {}
153
167
  end
@@ -0,0 +1,92 @@
1
+ require "aws-sdk-ec2"
2
+ require "aws-sdk-ecs"
3
+
4
+ module Wrapbox
5
+ module Runner
6
+ class Ecs
7
+ class InstanceManager
8
+ def initialize(cluster, region, launch_template:, instance_type: nil, tag_specifications: nil, wait_until_instance_terminated: true)
9
+ @cluster = cluster
10
+ @region = region
11
+ @launch_template = launch_template
12
+ @instance_type = instance_type
13
+ @tag_specifications = tag_specifications
14
+ @wait_until_instance_terminated = wait_until_instance_terminated
15
+ @queue = Queue.new
16
+ @instance_ids = []
17
+ end
18
+
19
+ def pop_ec2_instance_id
20
+ Wrapbox.logger.debug("Wait until a new container instance are registered in \"#{@cluster}\" cluster")
21
+ @queue.pop
22
+ end
23
+
24
+ def start_preparing_instances(count)
25
+ preparing_instance_ids = ec2_client.run_instances(
26
+ launch_template: @launch_template,
27
+ instance_type: @instance_type,
28
+ tag_specifications: @tag_specifications,
29
+ min_count: count,
30
+ max_count: count
31
+ ).instances.map(&:instance_id)
32
+ @instance_ids.concat(preparing_instance_ids)
33
+ ec2_client.wait_until(:instance_running, instance_ids: preparing_instance_ids)
34
+
35
+ waiter = Aws::Waiters::Waiter.new(
36
+ max_attempts: 40,
37
+ delay: 15,
38
+ poller: Aws::Waiters::Poller.new(
39
+ operation_name: :list_container_instances,
40
+ acceptors: [
41
+ {
42
+ "expected" => true,
43
+ "matcher" => "path",
44
+ "state" => "success",
45
+ "argument" => "length(container_instance_arns) > `0`"
46
+ }
47
+ ]
48
+ )
49
+ )
50
+
51
+ while preparing_instance_ids.size > 0
52
+ waiter.wait(client: ecs_client, params: { cluster: @cluster, filter: "ec2InstanceId in [#{preparing_instance_ids.join(",")}]" }).each do |resp|
53
+ ecs_client.describe_container_instances(cluster: @cluster, container_instances: resp.container_instance_arns).container_instances.each do |c|
54
+ preparing_instance_ids.delete(c.ec2_instance_id)
55
+ @queue << c.ec2_instance_id
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ def terminate_instance(instance_id)
62
+ ec2_client.terminate_instances(instance_ids: [instance_id])
63
+ if @wait_until_instance_terminated
64
+ ec2_client.wait_until(:instance_terminated, instance_ids: [instance_id])
65
+ end
66
+ @instance_ids.delete(instance_id)
67
+ end
68
+
69
+ def terminate_all_instances
70
+ # Duplicate @instance_ids because other threads can change it
71
+ remaining_instance_ids = @instance_ids.dup
72
+ return if remaining_instance_ids.empty?
73
+ ec2_client.terminate_instances(instance_ids: remaining_instance_ids)
74
+ if @wait_until_instance_terminated
75
+ ec2_client.wait_until(:instance_terminated, instance_ids: remaining_instance_ids)
76
+ end
77
+ @instance_ids.clear
78
+ end
79
+
80
+ private
81
+
82
+ def ecs_client
83
+ @ecs_client ||= Aws::ECS::Client.new({ region: @region }.reject { |_, v| v.nil? })
84
+ end
85
+
86
+ def ec2_client
87
+ @ec2_client ||= Aws::EC2::Client.new({ region: @region }.reject { |_, v| v.nil? })
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,120 @@
1
+ require "timeout"
2
+
3
+ require "aws-sdk-ecs"
4
+
5
+ module Wrapbox
6
+ module Runner
7
+ class Ecs
8
+ class TaskWaiter
9
+ MAX_DESCRIBABLE_TASK_COUNT = 100
10
+
11
+ class WaitFailure < StandardError; end
12
+ class TaskStopped < WaitFailure; end
13
+ class TaskMissing < WaitFailure; end
14
+ class UnknownFailure < WaitFailure; end
15
+ class WaitTimeout < WaitFailure; end
16
+
17
+ def initialize(cluster:, region:, delay:)
18
+ @cluster = cluster
19
+ @region = region
20
+ @task_arn_to_described_result = {}
21
+ @mutex = Mutex.new
22
+ @cv = ConditionVariable.new
23
+ Thread.new { update_described_results(delay) }
24
+ end
25
+
26
+ # @return Aws::ECS::Types::Task
27
+ def wait_task_running(task_arn, timeout: 0)
28
+ Timeout.timeout(timeout) do
29
+ loop do
30
+ result = describe_task(task_arn)
31
+ if result[:failure]
32
+ case result[:failure].reason
33
+ when "MISSING"
34
+ raise TaskMissing
35
+ else
36
+ raise UnknownFailure
37
+ end
38
+ end
39
+ raise TaskStopped if result[:task].last_status == "STOPPED"
40
+
41
+ return result[:task] if result[:task].last_status == "RUNNING"
42
+ end
43
+ end
44
+ rescue Timeout::Error
45
+ raise WaitTimeout
46
+ end
47
+
48
+ # @return Aws::ECS::Types::Task
49
+ def wait_task_stopped(task_arn, timeout: 0)
50
+ Timeout.timeout(timeout) do
51
+ result = nil
52
+ loop do
53
+ result = describe_task(task_arn)
54
+ if result[:failure]
55
+ case result[:failure].reason
56
+ when "MISSING"
57
+ raise TaskMissing
58
+ else
59
+ raise UnknownFailure
60
+ end
61
+ end
62
+
63
+ return result[:task] if result[:task].last_status == "STOPPED"
64
+ end
65
+ end
66
+ rescue Timeout::Error
67
+ raise WaitTimeout
68
+ end
69
+
70
+ private
71
+
72
+ def describe_task(task_arn)
73
+ @mutex.synchronize do
74
+ @task_arn_to_described_result[task_arn] = nil
75
+ @cv.wait(@mutex)
76
+ @task_arn_to_described_result[task_arn]
77
+ end
78
+ ensure
79
+ @mutex.synchronize do
80
+ @task_arn_to_described_result.delete(task_arn)
81
+ end
82
+ end
83
+
84
+ def update_described_results(interval)
85
+ loop do
86
+ @mutex.synchronize do
87
+ unless @task_arn_to_described_result.empty?
88
+ begin
89
+ @task_arn_to_described_result.keys.each_slice(MAX_DESCRIBABLE_TASK_COUNT) do |task_arns|
90
+ resp = ecs_client.describe_tasks(cluster: @cluster, tasks: task_arns)
91
+ resp.tasks.each do |task|
92
+ @task_arn_to_described_result[task.task_arn] = { task: task }
93
+ end
94
+ resp.failures.each do |failure|
95
+ # failure.arn is like "arn:aws:ecs:<region>:<account-id>:task/<task-id>"
96
+ # even if task_arn is like "arn:aws:ecs:<region>:<account-id>:task/<cluster>/<task-id>"
97
+ prefix, suffix = failure.arn.split("/")
98
+ task_arn = task_arns.find { |a| a.start_with?(prefix) && a.end_with?(suffix) }
99
+ @task_arn_to_described_result[task_arn || failure.arn] = { failure: failure }
100
+ end
101
+ end
102
+
103
+ @cv.broadcast
104
+ rescue Aws::ECS::Errors::ThrottlingException
105
+ Wrapbox.logger.warn("Failed to describe tasks due to Aws::ECS::Errors::ThrottlingException")
106
+ end
107
+ end
108
+ end
109
+
110
+ sleep interval
111
+ end
112
+ end
113
+
114
+ def ecs_client
115
+ @ecs_client ||= Aws::ECS::Client.new({ region: @region }.reject { |_, v| v.nil? })
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end