elastic_beans 0.10.0.alpha3 → 0.10.0.alpha4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/elastic_beans/application.rb +2 -1
- data/lib/elastic_beans/exec/command.rb +16 -2
- data/lib/elastic_beans/exec/sqs_consumer.rb +67 -29
- data/lib/elastic_beans/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab319bee66fb6845dab2666354b537f014d57922
|
4
|
+
data.tar.gz: 1a6f05149363a12674100472bba10579d3e18d00
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ef907a6ddabf1243e976d882b22d1524510816ff58690c8141f55866781b77295294aa3b9117d8800828072650e9733786ae0da9c3cd9830f1d1c16dbdb781e
|
7
|
+
data.tar.gz: c9c517314e4403cfe788ab28667f35a029d388feefaa6f8c3b8100258fb18e6082e4306ad7321a69def8ea3313e6b44577acda3dec2a8d873612b221b4a6ad40
|
@@ -149,7 +149,8 @@ module ElasticBeans
|
|
149
149
|
objects.each_with_object([]) { |object, commands|
|
150
150
|
begin
|
151
151
|
response = s3.get_object(bucket: bucket_name, key: object.key)
|
152
|
-
|
152
|
+
command = ElasticBeans::Exec::Command.from_json(response.body.read)
|
153
|
+
commands << command unless command.freeze_instance?
|
153
154
|
# skip finished or invalid commands
|
154
155
|
rescue ::Aws::S3::Errors::NoSuchKey
|
155
156
|
rescue JSON::ParserError
|
@@ -21,9 +21,10 @@ module ElasticBeans
|
|
21
21
|
# Command metadata, such as where to store execution data in S3.
|
22
22
|
attr_reader :metadata
|
23
23
|
|
24
|
-
def initialize(command_string:, id: nil, instance_id: nil, start_time: nil, metadata: nil)
|
24
|
+
def initialize(command_string:, freeze_instance: false, id: nil, instance_id: nil, start_time: nil, metadata: nil)
|
25
25
|
@id = id || SecureRandom.uuid
|
26
26
|
@command_string = command_string
|
27
|
+
@freeze = freeze_instance
|
27
28
|
@instance_id = instance_id
|
28
29
|
@start_time = start_time
|
29
30
|
@metadata = metadata || {}
|
@@ -33,10 +34,16 @@ module ElasticBeans
|
|
33
34
|
id == other.id
|
34
35
|
end
|
35
36
|
|
37
|
+
# Instead of a +command_string+ to execute, instruct the instance to not do anything.
|
38
|
+
def freeze_instance?
|
39
|
+
@freeze
|
40
|
+
end
|
41
|
+
|
36
42
|
def to_json
|
37
43
|
attributes = {}
|
38
44
|
attributes[:id] = id
|
39
|
-
attributes[:command] = command_string
|
45
|
+
attributes[:command] = command_string if command_string
|
46
|
+
attributes[:freeze] = true if freeze_instance?
|
40
47
|
attributes[:instance_id] = instance_id if instance_id
|
41
48
|
attributes[:start_time] = start_time.iso8601 if start_time
|
42
49
|
attributes[:metadata] = metadata
|
@@ -48,11 +55,18 @@ module ElasticBeans
|
|
48
55
|
new(
|
49
56
|
id: attributes["id"],
|
50
57
|
command_string: attributes["command"],
|
58
|
+
freeze_instance: attributes["freeze"],
|
51
59
|
instance_id: attributes["instance_id"],
|
52
60
|
start_time: attributes["start_time"] ? Time.iso8601(attributes["start_time"]) : nil,
|
53
61
|
metadata: attributes["metadata"],
|
54
62
|
)
|
55
63
|
end
|
64
|
+
|
65
|
+
# A special ElasticBeans::Exec::Command that instructs the instance to not do anything.
|
66
|
+
# Used for interactive exec commands, like +rails console+.
|
67
|
+
def self.freeze_instance
|
68
|
+
new(command_string: nil, freeze_instance: true)
|
69
|
+
end
|
56
70
|
end
|
57
71
|
end
|
58
72
|
end
|
@@ -7,6 +7,9 @@ require "time"
|
|
7
7
|
# @!visibility private
|
8
8
|
class SQSConsumer
|
9
9
|
LOOP_PERIOD = 1
|
10
|
+
KILL_PERIOD = 5
|
11
|
+
TERM_TIMEOUT = 25
|
12
|
+
FREEZE_TIMEOUT = 86400
|
10
13
|
|
11
14
|
def initialize(instance_id:, logdir:, queue_url:, s3:, sqs:)
|
12
15
|
@instance_id = instance_id
|
@@ -34,47 +37,50 @@ class SQSConsumer
|
|
34
37
|
next
|
35
38
|
end
|
36
39
|
|
37
|
-
log_command("Executing command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}...")
|
38
40
|
start_time = Time.now
|
39
|
-
|
40
|
-
"
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
if command['freeze']
|
42
|
+
log("Freezing command ID=#{command['id']} on host #{`hostname`.chomp}...")
|
43
|
+
else
|
44
|
+
log_command("Executing command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}...")
|
45
|
+
@command_pid = Process.spawn(
|
46
|
+
"#{command_script} #{command['command']}",
|
47
|
+
out: [command_logfile, "a"],
|
48
|
+
err: [command_logfile, "a"],
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
44
52
|
delete_message(message)
|
45
53
|
update_metadata(command: command, start_time: start_time)
|
46
54
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
55
|
+
if command['freeze']
|
56
|
+
begin
|
57
|
+
Timeout.timeout(FREEZE_TIMEOUT) do
|
58
|
+
while_killed(command) do
|
59
|
+
throw(:killed)
|
60
|
+
end
|
61
|
+
log("Defrosting from freeze command ID=#{command['id']}")
|
62
|
+
end
|
63
|
+
rescue Timeout::Error
|
64
|
+
log("Freeze command ID=#{command['id']} never terminated; timing out after '#{FREEZE_TIMEOUT}' seconds")
|
65
|
+
end
|
66
|
+
else
|
67
|
+
# poll command metadata to see if it is still available, if it disappears then kill the process
|
68
|
+
killed_thread = Thread.new do
|
69
|
+
while_killed(command) do
|
53
70
|
if already_killed?
|
54
|
-
|
55
|
-
|
56
|
-
" because it was killed and has not terminated yet"
|
57
|
-
)
|
58
|
-
Process.kill("KILL", command_pid)
|
59
|
-
break
|
71
|
+
kill_command(command, force: true)
|
72
|
+
throw(:killed)
|
60
73
|
else
|
61
|
-
|
62
|
-
"Sending TERM signal to command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}" \
|
63
|
-
" because it was killed"
|
64
|
-
)
|
65
|
-
Process.kill("TERM", command_pid)
|
74
|
+
kill_command(command)
|
66
75
|
@killed = true
|
67
|
-
# give the command a little bit to die
|
68
|
-
sleep 25
|
69
76
|
end
|
70
77
|
end
|
71
78
|
end
|
79
|
+
_, status = Process.wait2(command_pid)
|
80
|
+
killed_thread.kill if killed_thread.alive?
|
81
|
+
log_command("Command ID=#{command['id']} `#{command['command']}' exited on host #{`hostname`.chomp}: #{status}")
|
72
82
|
end
|
73
83
|
|
74
|
-
_, status = Process.wait2(command_pid)
|
75
|
-
killed_thread.kill if killed_thread.alive?
|
76
|
-
log_command("Command ID=#{command['id']} `#{command['command']}' exited on host #{`hostname`.chomp}: #{status}")
|
77
|
-
|
78
84
|
@command_pid = nil
|
79
85
|
@killed = nil
|
80
86
|
remove_metadata(command: command)
|
@@ -191,4 +197,36 @@ class SQSConsumer
|
|
191
197
|
rescue StandardError => e
|
192
198
|
log("[remove_metadata] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
|
193
199
|
end
|
200
|
+
|
201
|
+
def kill_command(command, force: false)
|
202
|
+
if force
|
203
|
+
signal = "KILL"
|
204
|
+
else
|
205
|
+
signal = "TERM"
|
206
|
+
end
|
207
|
+
|
208
|
+
log_command(
|
209
|
+
"Sending signal '#{signal}' to" \
|
210
|
+
" command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}" \
|
211
|
+
" because it was killed"
|
212
|
+
)
|
213
|
+
Process.kill(signal, command_pid)
|
214
|
+
|
215
|
+
unless force
|
216
|
+
# give the command a little bit to die
|
217
|
+
sleep(TERM_TIMEOUT)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
def while_killed(command)
|
222
|
+
catch(:killed) do
|
223
|
+
loop do
|
224
|
+
sleep(KILL_PERIOD)
|
225
|
+
|
226
|
+
if !metadata_exists?(command: command)
|
227
|
+
yield if block_given?
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
194
232
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elastic_beans
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.0.
|
4
|
+
version: 0.10.0.alpha4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Stegman
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-04-
|
11
|
+
date: 2017-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|