elastic_beans 0.13.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +0 -5
- data/lib/elastic_beans/cli.rb +4 -1
- data/lib/elastic_beans/command/configure.rb +140 -69
- data/lib/elastic_beans/exec/elastic_beans_exec.conf +24 -1
- data/lib/elastic_beans/exec/init.rb +53 -13
- data/lib/elastic_beans/exec/sqs_consumer.rb +32 -12
- data/lib/elastic_beans/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 85d2e1b5cb7fb1449bf1c31ef6486324f54e554f
|
4
|
+
data.tar.gz: 9efe208d0b3ffcde0e6b57c376af44cca66da2c3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f69ee93f26ee6f40a15841600d2bcd30ad5c3afe4d5b51469b163d7ce093f95379fa377eeedcd0f2fad729ae584c740bc9777c64131c9e21f184038590f37160
|
7
|
+
data.tar.gz: d41f1f33cc3dd28ff8f51748e09a82960343c1c4a22178dbf31ded0fc3e4bbf76cca23bb21cb1e09c4773d8fff9c1cfea5cf7a8ef195ee9814a1ef8450c19bdd
|
data/README.md
CHANGED
@@ -261,11 +261,6 @@ Make sure that its redrive policy allows such inspection before considering a me
|
|
261
261
|
Your application must use the [active-elastic-job gem](https://github.com/tawan/active-elastic-job) for background job processing.
|
262
262
|
Elastic Beans will set the `PROCESS_ACTIVE_ELASTIC_JOBS` environment variable appropriately in your environments.
|
263
263
|
|
264
|
-
### Known issues and limitations of the pre-release
|
265
|
-
|
266
|
-
Elastic Beans is in pre-release and still has some rough edges that need to be worked out.
|
267
|
-
See [the wiki page](https://github.com/onemedical/elastic_beans/wiki/Known-Issues).
|
268
|
-
|
269
264
|
## Installation
|
270
265
|
|
271
266
|
Add this line to your application's Gemfile:
|
data/lib/elastic_beans/cli.rb
CHANGED
@@ -23,7 +23,8 @@ class ElasticBeans::CLI < Thor
|
|
23
23
|
option :public_key, aliases: %w(-p), desc: "For end-to-end encryption. The public key of the SSL certificate the ELB will verify to communicate with your Rails app"
|
24
24
|
option :ssl_certificate_arn, aliases: %w(-s), desc: "The ARN of the SSL server certificate stored in IAM to attach to the ELB"
|
25
25
|
option :solution_stack, desc: "Solution stack name to deploy the application on. Defaults to the latest Ruby Puma solution stack."
|
26
|
-
|
26
|
+
option :queue, default: "default", aliases: %w(-q), desc: "The name of the queue a new worker environment should listen to, e.g. `default`"
|
27
|
+
def configure(environment_type = nil)
|
27
28
|
@verbose = options[:verbose]
|
28
29
|
ElasticBeans::Command::Configure.new(
|
29
30
|
internal: options[:internal],
|
@@ -38,6 +39,8 @@ class ElasticBeans::CLI < Thor
|
|
38
39
|
elastic_beanstalk: elastic_beanstalk_client,
|
39
40
|
iam: iam_client,
|
40
41
|
ui: ui,
|
42
|
+
environment_type: environment_type,
|
43
|
+
queue: options[:queue]
|
41
44
|
).run
|
42
45
|
rescue StandardError => e
|
43
46
|
error(e)
|
@@ -9,11 +9,13 @@ module ElasticBeans
|
|
9
9
|
DESC = "Configure the given Elastic Beanstalk application"
|
10
10
|
LONG_DESC = <<-LONG_DESC
|
11
11
|
Configure the given Elastic Beanstalk application.
|
12
|
+
Specify an environment to configure a single environment within your EB application
|
12
13
|
Creates and updates configuration templates for each possible environment type.
|
13
14
|
Updates running environments with the new configuration.
|
14
15
|
|
15
16
|
Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
|
16
17
|
LONG_DESC
|
18
|
+
VALID_ARGUMENTS = %w(webserver worker exec scheduler)
|
17
19
|
|
18
20
|
def initialize(
|
19
21
|
internal:,
|
@@ -27,7 +29,9 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
27
29
|
network:,
|
28
30
|
elastic_beanstalk:,
|
29
31
|
iam:,
|
30
|
-
ui
|
32
|
+
ui:,
|
33
|
+
environment_type: nil,
|
34
|
+
queue:
|
31
35
|
)
|
32
36
|
@internal = internal
|
33
37
|
@keypair = keypair
|
@@ -41,16 +45,15 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
41
45
|
@elastic_beanstalk = elastic_beanstalk
|
42
46
|
@iam = iam
|
43
47
|
@ui = ui
|
48
|
+
@environment_type = environment_type
|
49
|
+
@queue = queue
|
50
|
+
# Required attributes not passed from CLI
|
51
|
+
@environments = application.environments
|
52
|
+
@threads = []
|
44
53
|
end
|
45
54
|
|
46
55
|
def run
|
47
|
-
|
48
|
-
unready_environments = environments.select { |environment| environment.status != "Ready" }
|
49
|
-
if unready_environments.any?
|
50
|
-
raise EnvironmentsNotReady.new(environments: unready_environments)
|
51
|
-
end
|
52
|
-
|
53
|
-
threads = []
|
56
|
+
check_environment_status
|
54
57
|
|
55
58
|
Signal.trap("INT") do
|
56
59
|
puts "\nInterrupting beans"
|
@@ -63,24 +66,112 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
63
66
|
raise SignalException.new("TERM")
|
64
67
|
end
|
65
68
|
|
66
|
-
|
67
|
-
progressbar = ProgressBar.create(title: "Configuring", total: config_templates, output: ui.stdout)
|
69
|
+
configure_progressbar
|
68
70
|
progressbar.log("Updating configuration templates in #{application.name}...")
|
69
|
-
progressbar.log("Updating base configuration template...")
|
70
|
-
base_config = ElasticBeans::ConfigurationTemplate::Base.new(
|
71
|
-
application: application,
|
72
|
-
elastic_beanstalk: elastic_beanstalk,
|
73
|
-
)
|
74
|
-
base_config.upsert(
|
75
|
-
network: network,
|
76
|
-
keypair: keypair,
|
77
|
-
solution_stack: solution_stack,
|
78
|
-
option_settings: option_settings,
|
79
|
-
options_to_remove: options_to_remove,
|
80
|
-
iam: iam,
|
81
|
-
)
|
82
|
-
progressbar.increment
|
83
71
|
|
72
|
+
if environment_type
|
73
|
+
raise InvalidCommandArgumentError if !VALID_ARGUMENTS.include?(environment_type)
|
74
|
+
self.send("configure_#{environment_type}".to_sym)
|
75
|
+
else
|
76
|
+
# Shared configuration with common values for all environments
|
77
|
+
progressbar.log("Updating base configuration template...")
|
78
|
+
base_config = ElasticBeans::ConfigurationTemplate::Base.new(
|
79
|
+
application: application,
|
80
|
+
elastic_beanstalk: elastic_beanstalk,
|
81
|
+
)
|
82
|
+
base_config.upsert(
|
83
|
+
network: network,
|
84
|
+
keypair: keypair,
|
85
|
+
solution_stack: solution_stack,
|
86
|
+
option_settings: option_settings,
|
87
|
+
options_to_remove: options_to_remove,
|
88
|
+
iam: iam,
|
89
|
+
)
|
90
|
+
progressbar.increment
|
91
|
+
|
92
|
+
configure_webserver
|
93
|
+
configure_exec
|
94
|
+
configure_scheduler
|
95
|
+
configure_worker
|
96
|
+
end
|
97
|
+
|
98
|
+
threads.each(&:join)
|
99
|
+
progressbar.finish
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
attr_reader(
|
105
|
+
:application,
|
106
|
+
:internal,
|
107
|
+
:keypair,
|
108
|
+
:network,
|
109
|
+
:public_key,
|
110
|
+
:ssl_certificate_arn,
|
111
|
+
:solution_stack,
|
112
|
+
:elastic_beanstalk,
|
113
|
+
:iam,
|
114
|
+
:ui,
|
115
|
+
:environment_type,
|
116
|
+
:queue,
|
117
|
+
:progressbar,
|
118
|
+
:environments,
|
119
|
+
:threads
|
120
|
+
)
|
121
|
+
|
122
|
+
# Coerces +@option_setting_strings+ into object format.
|
123
|
+
def option_settings
|
124
|
+
@option_settings ||= @option_setting_strings.map { |option_setting_string|
|
125
|
+
setting, value = option_setting_string.split('=', 2)
|
126
|
+
if !setting || setting.empty? || !value || value.empty?
|
127
|
+
raise InvalidOptionSettingFormatError
|
128
|
+
end
|
129
|
+
namespace, option_name = setting.split('/', 2)
|
130
|
+
if !namespace || namespace.empty? || !option_name || option_name.empty?
|
131
|
+
raise InvalidOptionSettingFormatError
|
132
|
+
end
|
133
|
+
{namespace: namespace, option_name: option_name, value: value}
|
134
|
+
}
|
135
|
+
end
|
136
|
+
|
137
|
+
# Coerces +@option_strings_to_remove+ into object format.
|
138
|
+
def options_to_remove
|
139
|
+
@options_to_remove ||= @option_strings_to_remove.map { |option_setting_string|
|
140
|
+
namespace, option_name = option_setting_string.split('/', 2)
|
141
|
+
if !namespace || namespace.empty? || !option_name || option_name.empty? || option_name.include?('=')
|
142
|
+
raise InvalidOptionSettingFormatError
|
143
|
+
end
|
144
|
+
{namespace: namespace, option_name: option_name}
|
145
|
+
}
|
146
|
+
end
|
147
|
+
|
148
|
+
def check_environment_status
|
149
|
+
target_environments = if environment_type && environment_type != 'worker'
|
150
|
+
environments.select { |environment| environment.name.include?(environment_type) }
|
151
|
+
elsif environment_type == 'worker'
|
152
|
+
environments.select { |environment| environment.name.include?(environment_type) && environment.name.include?(queue) }
|
153
|
+
else
|
154
|
+
environments
|
155
|
+
end
|
156
|
+
|
157
|
+
unready_environments = target_environments.select { |environment| environment.status != "Ready" }
|
158
|
+
if unready_environments.any?
|
159
|
+
raise EnvironmentsNotReady.new(environments: unready_environments)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def configure_progressbar
|
164
|
+
config_templates = if !environment_type
|
165
|
+
# (Base + exec + scheduler + webserver) + total number of worker queues
|
166
|
+
4 + application.worker_queues.size
|
167
|
+
else
|
168
|
+
1
|
169
|
+
end
|
170
|
+
|
171
|
+
@progressbar = ProgressBar.create(title: "Configuring", total: config_templates, output: ui.stdout)
|
172
|
+
end
|
173
|
+
|
174
|
+
def configure_webserver
|
84
175
|
progressbar.log("Updating webserver configuration template...")
|
85
176
|
webserver_config = ElasticBeans::ConfigurationTemplate::Webserver.new(
|
86
177
|
application: application,
|
@@ -105,7 +196,9 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
105
196
|
end
|
106
197
|
end
|
107
198
|
progressbar.increment
|
199
|
+
end
|
108
200
|
|
201
|
+
def configure_exec
|
109
202
|
progressbar.log("Updating exec configuration template...")
|
110
203
|
exec_config = ElasticBeans::ConfigurationTemplate::Exec.new(
|
111
204
|
application: application,
|
@@ -127,7 +220,9 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
127
220
|
end
|
128
221
|
end
|
129
222
|
progressbar.increment
|
223
|
+
end
|
130
224
|
|
225
|
+
def configure_scheduler
|
131
226
|
progressbar.log("Updating scheduler configuration template...")
|
132
227
|
scheduler_config = ElasticBeans::ConfigurationTemplate::Scheduler.new(
|
133
228
|
application: application,
|
@@ -149,8 +244,12 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
149
244
|
end
|
150
245
|
end
|
151
246
|
progressbar.increment
|
247
|
+
end
|
152
248
|
|
153
|
-
|
249
|
+
def configure_worker
|
250
|
+
target_queues = environment_type ? application.worker_queues.select { |q| q == queue } : application.worker_queues
|
251
|
+
|
252
|
+
target_queues.each do |queue|
|
154
253
|
progressbar.log("Updating worker-#{queue} configuration template...")
|
155
254
|
worker_config = ElasticBeans::ConfigurationTemplate::Worker.new(
|
156
255
|
queue: queue,
|
@@ -173,50 +272,6 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
|
|
173
272
|
end
|
174
273
|
end
|
175
274
|
end
|
176
|
-
threads.each(&:join)
|
177
|
-
progressbar.increment
|
178
|
-
progressbar.finish
|
179
|
-
end
|
180
|
-
|
181
|
-
private
|
182
|
-
|
183
|
-
attr_reader(
|
184
|
-
:application,
|
185
|
-
:internal,
|
186
|
-
:keypair,
|
187
|
-
:network,
|
188
|
-
:public_key,
|
189
|
-
:ssl_certificate_arn,
|
190
|
-
:solution_stack,
|
191
|
-
:elastic_beanstalk,
|
192
|
-
:iam,
|
193
|
-
:ui,
|
194
|
-
)
|
195
|
-
|
196
|
-
# Coerces +@option_setting_strings+ into object format.
|
197
|
-
def option_settings
|
198
|
-
@option_settings ||= @option_setting_strings.map { |option_setting_string|
|
199
|
-
setting, value = option_setting_string.split('=', 2)
|
200
|
-
if !setting || setting.empty? || !value || value.empty?
|
201
|
-
raise InvalidOptionSettingFormatError
|
202
|
-
end
|
203
|
-
namespace, option_name = setting.split('/', 2)
|
204
|
-
if !namespace || namespace.empty? || !option_name || option_name.empty?
|
205
|
-
raise InvalidOptionSettingFormatError
|
206
|
-
end
|
207
|
-
{namespace: namespace, option_name: option_name, value: value}
|
208
|
-
}
|
209
|
-
end
|
210
|
-
|
211
|
-
# Coerces +@option_strings_to_remove+ into object format.
|
212
|
-
def options_to_remove
|
213
|
-
@options_to_remove ||= @option_strings_to_remove.map { |option_setting_string|
|
214
|
-
namespace, option_name = option_setting_string.split('/', 2)
|
215
|
-
if !namespace || namespace.empty? || !option_name || option_name.empty? || option_name.include?('=')
|
216
|
-
raise InvalidOptionSettingFormatError
|
217
|
-
end
|
218
|
-
{namespace: namespace, option_name: option_name}
|
219
|
-
}
|
220
275
|
end
|
221
276
|
|
222
277
|
# :nodoc: all
|
@@ -248,6 +303,22 @@ Option settings to remove should in the format 'NAMESPACE/OPTION_NAME', e.g. 'aw
|
|
248
303
|
If you'd like to *set* option settings, use --option-settings 'NAMESPACE/OPTION_NAME=VALUE'.
|
249
304
|
Please re-run `#{command_as_string "configure"}` with updated syntax.
|
250
305
|
|
306
|
+
#{command_help "configure"}
|
307
|
+
MESSAGE
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
# :nodoc: all
|
312
|
+
# @!visibility private
|
313
|
+
class InvalidCommandArgumentError < ElasticBeans::Error
|
314
|
+
def message
|
315
|
+
require "elastic_beans/cli"
|
316
|
+
require "elastic_beans/cli/string_shell"
|
317
|
+
<<-MESSAGE
|
318
|
+
Unknown environment '#{@environment_type}'
|
319
|
+
Command 'configure' can be run without passing an argument or by using one of the following: #{VALID_ARGUMENTS}
|
320
|
+
Please re-run `#{command_as_string "configure"}` with updated syntax.
|
321
|
+
|
251
322
|
#{command_help "configure"}
|
252
323
|
MESSAGE
|
253
324
|
end
|
@@ -9,7 +9,30 @@ script
|
|
9
9
|
source $EB_SUPPORT_DIR/envvars
|
10
10
|
|
11
11
|
# execute using Elastic Beanstalk's built-in Ruby with aws-sdk pre-installed
|
12
|
+
ruby_version="`ls /opt/elasticbeanstalk/lib/ruby/lib/ruby/ | grep '^[0-9]\.[0-9]'`"
|
12
13
|
export PATH="/opt/elasticbeanstalk/lib/ruby/bin:$PATH"
|
13
|
-
export GEM_PATH="/opt/elasticbeanstalk/lib/ruby/lib/ruby
|
14
|
+
export GEM_PATH="/opt/elasticbeanstalk/lib/ruby/lib/ruby/$ruby_version"
|
14
15
|
exec ruby /opt/elastic_beans/exec/init.rb start
|
15
16
|
end script
|
17
|
+
|
18
|
+
pre-start script
|
19
|
+
EB_SUPPORT_DIR="`/opt/elasticbeanstalk/bin/get-config container -k support_dir`"
|
20
|
+
source $EB_SUPPORT_DIR/envvars
|
21
|
+
|
22
|
+
# execute using Elastic Beanstalk's built-in Ruby with aws-sdk pre-installed
|
23
|
+
ruby_version="`ls /opt/elasticbeanstalk/lib/ruby/lib/ruby/ | grep '^[0-9]\.[0-9]'`"
|
24
|
+
export PATH="/opt/elasticbeanstalk/lib/ruby/bin:$PATH"
|
25
|
+
export GEM_PATH="/opt/elasticbeanstalk/lib/ruby/lib/ruby/$ruby_version"
|
26
|
+
exec ruby /opt/elastic_beans/exec/init.rb pre_start
|
27
|
+
end script
|
28
|
+
|
29
|
+
post-stop script
|
30
|
+
EB_SUPPORT_DIR="`/opt/elasticbeanstalk/bin/get-config container -k support_dir`"
|
31
|
+
source $EB_SUPPORT_DIR/envvars
|
32
|
+
|
33
|
+
# execute using Elastic Beanstalk's built-in Ruby with aws-sdk pre-installed
|
34
|
+
ruby_version="`ls /opt/elasticbeanstalk/lib/ruby/lib/ruby/ | grep '^[0-9]\.[0-9]'`"
|
35
|
+
export PATH="/opt/elasticbeanstalk/lib/ruby/bin:$PATH"
|
36
|
+
export GEM_PATH="/opt/elasticbeanstalk/lib/ruby/lib/ruby/$ruby_version"
|
37
|
+
exec ruby /opt/elastic_beans/exec/init.rb post_stop
|
38
|
+
end script
|
@@ -1,11 +1,13 @@
|
|
1
1
|
require "aws-sdk"
|
2
2
|
require "fileutils"
|
3
3
|
require "net/http"
|
4
|
+
require "timeout"
|
4
5
|
require "uri"
|
5
6
|
|
6
7
|
# :nodoc: all
|
7
8
|
# @!visibility private
|
8
9
|
class Init
|
10
|
+
APP_DIR_TIMEOUT = 1800
|
9
11
|
LOGDIR = "/var/log/elastic_beans/exec"
|
10
12
|
PIDFILE = "/var/run/elastic_beans_exec.pid"
|
11
13
|
AZ_PATTERN = /\A(?<region>.+-\d+)\w+\z/
|
@@ -15,35 +17,54 @@ class Init
|
|
15
17
|
def self.run(command)
|
16
18
|
init = new
|
17
19
|
unless init.respond_to?(command)
|
18
|
-
abort "Usage: #{$0} {start}"
|
20
|
+
abort "Usage: #{$0} {start|pre_start|post_stop}"
|
19
21
|
end
|
20
22
|
init.send(command)
|
21
23
|
end
|
22
24
|
|
23
25
|
def start
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
wait_until_app_dir_exists
|
27
|
+
consumer.run
|
28
|
+
end
|
29
|
+
|
30
|
+
def post_stop
|
31
|
+
consumer.cleanup
|
32
|
+
end
|
33
|
+
|
34
|
+
def pre_start
|
35
|
+
consumer.cleanup
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def app_dir
|
41
|
+
`/opt/elasticbeanstalk/bin/get-config container -k app_deploy_dir`
|
42
|
+
end
|
43
|
+
|
44
|
+
def consumer
|
45
|
+
args = {
|
32
46
|
instance_id: instance_id,
|
33
47
|
logdir: LOGDIR,
|
34
48
|
queue_url: queue_url,
|
35
49
|
s3: s3_client,
|
36
50
|
sqs: sqs_client,
|
37
|
-
|
38
|
-
|
51
|
+
}
|
52
|
+
require File.expand_path("../sqs_consumer", __FILE__)
|
53
|
+
consumer = SQSConsumer.new(**args)
|
39
54
|
end
|
40
55
|
|
41
|
-
private
|
42
|
-
|
43
56
|
def instance_id
|
44
57
|
@instance_id ||= Net::HTTP.get(INSTANCE_ID_URI)
|
45
58
|
end
|
46
59
|
|
60
|
+
def queue_url
|
61
|
+
queue_url = ENV['ELASTIC_BEANS_EXEC_QUEUE_URL']
|
62
|
+
if queue_url.nil?
|
63
|
+
raise "ELASTIC_BEANS_EXEC_QUEUE_URL not set; please re-run `beans configure`."
|
64
|
+
end
|
65
|
+
queue_url
|
66
|
+
end
|
67
|
+
|
47
68
|
def region
|
48
69
|
return @region if @region
|
49
70
|
az = Net::HTTP.get(AZ_URI)
|
@@ -53,6 +74,25 @@ class Init
|
|
53
74
|
end
|
54
75
|
@region = az_match[:region]
|
55
76
|
end
|
77
|
+
|
78
|
+
def s3_client
|
79
|
+
@s3_client ||= ::Aws::S3::Client.new(region: region)
|
80
|
+
end
|
81
|
+
|
82
|
+
def sqs_client
|
83
|
+
@sqs_client ||= ::Aws::SQS::Client.new(region: region)
|
84
|
+
end
|
85
|
+
|
86
|
+
def wait_until_app_dir_exists
|
87
|
+
::Timeout.timeout(APP_DIR_TIMEOUT) do
|
88
|
+
loop do
|
89
|
+
sleep 1
|
90
|
+
break if Dir.glob("#{app_dir}/*").any?
|
91
|
+
end
|
92
|
+
end
|
93
|
+
rescue ::Timeout::Error
|
94
|
+
raise "App dir #{app_dir} has no contents; waited #{APP_DIR_TIMEOUT} seconds"
|
95
|
+
end
|
56
96
|
end
|
57
97
|
|
58
98
|
Init.run(ARGV[0])
|
@@ -15,6 +15,7 @@ class SQSConsumer
|
|
15
15
|
@instance_id = instance_id
|
16
16
|
@logdir = logdir
|
17
17
|
@command_logfile = File.join(logdir, "command.log")
|
18
|
+
@command_metafile = File.join(logdir, "command.json")
|
18
19
|
@consumer_logfile = File.join(logdir, "sqs_consumer.log")
|
19
20
|
@queue_url = queue_url
|
20
21
|
@s3 = s3
|
@@ -93,18 +94,27 @@ class SQSConsumer
|
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
97
|
+
def cleanup
|
98
|
+
command = interrupted_command
|
99
|
+
if command
|
100
|
+
log("Cleaning up metadata for command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp}...")
|
101
|
+
remove_metadata(command: command)
|
102
|
+
else
|
103
|
+
log("No cleanup necessary on host #{`hostname`.chomp}")
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
96
107
|
private
|
97
108
|
|
98
|
-
attr_reader :consumer_logfile, :command_logfile, :command_script, :instance_id, :logdir, :queue_url, :s3, :sqs
|
109
|
+
attr_reader :consumer_logfile, :command_logfile, :command_metafile, :command_script, :instance_id, :logdir, :queue_url, :s3, :sqs
|
99
110
|
# Transient attributes
|
100
111
|
attr_reader :command_pid
|
101
112
|
|
102
|
-
def stop(signal =
|
103
|
-
log("Terminating message receiving process")
|
104
|
-
signal
|
105
|
-
|
113
|
+
def stop(signal = 15)
|
114
|
+
log("Terminating message receiving process with signal '#{signal}'")
|
115
|
+
force = (signal == 9)
|
116
|
+
kill_command(interrupted_command, force: force) if command_pid
|
106
117
|
exit
|
107
|
-
rescue Errno::ESRCH
|
108
118
|
end
|
109
119
|
|
110
120
|
def command_from_message(message)
|
@@ -114,6 +124,14 @@ class SQSConsumer
|
|
114
124
|
raise e
|
115
125
|
end
|
116
126
|
|
127
|
+
def interrupted_command
|
128
|
+
return unless File.exists?(command_metafile)
|
129
|
+
JSON.parse(File.read(command_metafile))
|
130
|
+
rescue StandardError => e
|
131
|
+
log("[interrupted_command] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
|
132
|
+
raise e
|
133
|
+
end
|
134
|
+
|
117
135
|
def log(message)
|
118
136
|
File.open(consumer_logfile, "a") do |file|
|
119
137
|
file.puts("[#{Time.now.iso8601}] [elastic_beans/exec] #{message}")
|
@@ -150,7 +168,7 @@ class SQSConsumer
|
|
150
168
|
rescue ::Aws::S3::Errors::NotFound
|
151
169
|
false
|
152
170
|
rescue StandardError => e
|
153
|
-
log("[
|
171
|
+
log("[killing_command?] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
|
154
172
|
end
|
155
173
|
|
156
174
|
def kill_command(command, force: false)
|
@@ -160,17 +178,17 @@ class SQSConsumer
|
|
160
178
|
signal = "TERM"
|
161
179
|
end
|
162
180
|
|
163
|
-
|
164
|
-
"
|
165
|
-
|
166
|
-
" because it was killed"
|
167
|
-
)
|
181
|
+
if command
|
182
|
+
log_command("Command ID=#{command['id']} `#{command['command']}' killed on host #{`hostname`.chomp}: pid #{command_pid} exit #{signal}")
|
183
|
+
end
|
168
184
|
Process.kill(signal, command_pid)
|
169
185
|
|
170
186
|
unless force
|
171
187
|
# give the command a little bit to die
|
172
188
|
sleep(TERM_TIMEOUT)
|
173
189
|
end
|
190
|
+
rescue Errno::ESRCH => e
|
191
|
+
log("[kill_command] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
|
174
192
|
end
|
175
193
|
|
176
194
|
def while_killed(command)
|
@@ -204,6 +222,7 @@ class SQSConsumer
|
|
204
222
|
end
|
205
223
|
|
206
224
|
def update_metadata(command:, start_time:)
|
225
|
+
File.write(command_metafile, command.to_json)
|
207
226
|
metadata = command['metadata']
|
208
227
|
if metadata && metadata['bucket'] && metadata['key']
|
209
228
|
s3.put_object(
|
@@ -231,6 +250,7 @@ class SQSConsumer
|
|
231
250
|
key: "#{metadata['key']}.killed",
|
232
251
|
)
|
233
252
|
end
|
253
|
+
FileUtils.rm_f(command_metafile)
|
234
254
|
rescue StandardError => e
|
235
255
|
log("[remove_metadata] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
|
236
256
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastic_beans
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Stegman
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
@@ -256,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
256
256
|
version: '0'
|
257
257
|
requirements: []
|
258
258
|
rubyforge_project:
|
259
|
-
rubygems_version: 2.
|
259
|
+
rubygems_version: 2.5.1
|
260
260
|
signing_key:
|
261
261
|
specification_version: 4
|
262
262
|
summary: Elastic Beanstalk environment orchestration for a Rails app
|