automan 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rvmrc +1 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +29 -0
- data/Rakefile +6 -0
- data/automan.gemspec +30 -0
- data/bin/baker +4 -0
- data/bin/mover +4 -0
- data/bin/scanner +4 -0
- data/bin/snapper +4 -0
- data/bin/stacker +4 -0
- data/bin/stalker +4 -0
- data/lib/automan.rb +27 -0
- data/lib/automan/base.rb +64 -0
- data/lib/automan/beanstalk/application.rb +74 -0
- data/lib/automan/beanstalk/configuration.rb +137 -0
- data/lib/automan/beanstalk/deployer.rb +193 -0
- data/lib/automan/beanstalk/errors.rb +22 -0
- data/lib/automan/beanstalk/package.rb +39 -0
- data/lib/automan/beanstalk/router.rb +102 -0
- data/lib/automan/beanstalk/terminator.rb +60 -0
- data/lib/automan/beanstalk/uploader.rb +58 -0
- data/lib/automan/beanstalk/version.rb +100 -0
- data/lib/automan/chef/uploader.rb +30 -0
- data/lib/automan/cli/baker.rb +63 -0
- data/lib/automan/cli/base.rb +14 -0
- data/lib/automan/cli/mover.rb +47 -0
- data/lib/automan/cli/scanner.rb +24 -0
- data/lib/automan/cli/snapper.rb +78 -0
- data/lib/automan/cli/stacker.rb +106 -0
- data/lib/automan/cli/stalker.rb +279 -0
- data/lib/automan/cloudformation/errors.rb +40 -0
- data/lib/automan/cloudformation/launcher.rb +196 -0
- data/lib/automan/cloudformation/replacer.rb +102 -0
- data/lib/automan/cloudformation/terminator.rb +61 -0
- data/lib/automan/cloudformation/uploader.rb +57 -0
- data/lib/automan/ec2/errors.rb +4 -0
- data/lib/automan/ec2/image.rb +137 -0
- data/lib/automan/ec2/instance.rb +83 -0
- data/lib/automan/mixins/aws_caller.rb +115 -0
- data/lib/automan/mixins/utils.rb +18 -0
- data/lib/automan/rds/errors.rb +7 -0
- data/lib/automan/rds/snapshot.rb +244 -0
- data/lib/automan/s3/downloader.rb +25 -0
- data/lib/automan/s3/uploader.rb +20 -0
- data/lib/automan/version.rb +3 -0
- data/lib/automan/wait_rescuer.rb +17 -0
- data/spec/beanstalk/application_spec.rb +49 -0
- data/spec/beanstalk/configuration_spec.rb +98 -0
- data/spec/beanstalk/deployer_spec.rb +162 -0
- data/spec/beanstalk/package_spec.rb +9 -0
- data/spec/beanstalk/router_spec.rb +65 -0
- data/spec/beanstalk/terminator_spec.rb +67 -0
- data/spec/beanstalk/uploader_spec.rb +53 -0
- data/spec/beanstalk/version_spec.rb +60 -0
- data/spec/chef/uploader_spec.rb +9 -0
- data/spec/cloudformation/launcher_spec.rb +240 -0
- data/spec/cloudformation/replacer_spec.rb +58 -0
- data/spec/cloudformation/templates/worker_role.json +337 -0
- data/spec/cloudformation/terminator_spec.rb +63 -0
- data/spec/cloudformation/uploader_spec.rb +50 -0
- data/spec/ec2/image_spec.rb +158 -0
- data/spec/ec2/instance_spec.rb +57 -0
- data/spec/mixins/aws_caller_spec.rb +39 -0
- data/spec/mixins/utils_spec.rb +44 -0
- data/spec/rds/snapshot_spec.rb +152 -0
- metadata +278 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
require 'automan'
|
|
2
|
+
|
|
3
|
+
describe Automan::Beanstalk::Uploader do
|
|
4
|
+
subject(:u) do
|
|
5
|
+
AWS.stub!
|
|
6
|
+
u = Automan::Beanstalk::Uploader.new
|
|
7
|
+
u.logger = Logger.new('/dev/null')
|
|
8
|
+
u.template_files = 'foo'
|
|
9
|
+
u.stub(:config_templates).and_return(['foo'])
|
|
10
|
+
u
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it { should respond_to :template_files }
|
|
14
|
+
it { should respond_to :s3_path }
|
|
15
|
+
it { should respond_to :upload_config_templates }
|
|
16
|
+
it { should respond_to :config_templates_valid? }
|
|
17
|
+
|
|
18
|
+
describe '#config_templates_valid?' do
|
|
19
|
+
|
|
20
|
+
it 'raises error if config templates do not exist' do
|
|
21
|
+
u.stub(:config_templates).and_return([])
|
|
22
|
+
expect {
|
|
23
|
+
u.config_templates_valid?
|
|
24
|
+
}.to raise_error(Automan::Beanstalk::NoConfigurationTemplatesError)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it 'returns true if config templates are valid json' do
|
|
28
|
+
u.stub(:read_config_template).and_return('[{}]')
|
|
29
|
+
u.config_templates_valid?.should be_true
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it 'returns false if config templates are invalid json' do
|
|
33
|
+
u.stub(:read_config_template).and_return('@#$%#')
|
|
34
|
+
u.config_templates_valid?.should be_false
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
describe '#upload_config_templates' do
|
|
39
|
+
it 'raises error if any config template fails validation' do
|
|
40
|
+
u.stub(:config_templates_valid?).and_return(false)
|
|
41
|
+
expect {
|
|
42
|
+
u.upload_config_templates
|
|
43
|
+
}.to raise_error(Automan::Beanstalk::InvalidConfigurationTemplateError)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
it 'uploads files if they are valid' do
|
|
47
|
+
u.stub(:config_templates).and_return(%w[a b c])
|
|
48
|
+
u.stub(:config_templates_valid?).and_return(true)
|
|
49
|
+
u.should_receive(:upload_file).exactly(3).times
|
|
50
|
+
u.upload_config_templates
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
require "automan"
|
|
2
|
+
|
|
3
|
+
describe Automan::Beanstalk::Version do
|
|
4
|
+
|
|
5
|
+
it { should respond_to :application }
|
|
6
|
+
it { should respond_to :label }
|
|
7
|
+
it { should respond_to :create }
|
|
8
|
+
it { should respond_to :delete }
|
|
9
|
+
it { should respond_to :cull_versions }
|
|
10
|
+
it { should respond_to :delete_by_label }
|
|
11
|
+
|
|
12
|
+
describe '#exists?' do
|
|
13
|
+
subject(:v) do
|
|
14
|
+
AWS.stub!
|
|
15
|
+
v = Automan::Beanstalk::Version.new
|
|
16
|
+
v.eb = AWS::ElasticBeanstalk::Client.new
|
|
17
|
+
v.application = 'foo'
|
|
18
|
+
v.label = 'v4'
|
|
19
|
+
v.log_aws_calls = false
|
|
20
|
+
v.logger = Logger.new('/dev/null')
|
|
21
|
+
v
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it "is true when version exists in response" do
|
|
25
|
+
resp = v.eb.stub_for :describe_application_versions
|
|
26
|
+
resp.data[:application_versions] = [
|
|
27
|
+
{application_name: v.application, version_label: v.label}
|
|
28
|
+
]
|
|
29
|
+
v.exists?.should be_true
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it "is false when version is not in response" do
|
|
33
|
+
resp = v.eb.stub_for :describe_application_versions
|
|
34
|
+
resp.data[:application_versions] = [
|
|
35
|
+
{application_name: '', version_label: ''}
|
|
36
|
+
]
|
|
37
|
+
v.exists?.should be_false
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
describe '#delete_by_label' do
|
|
42
|
+
subject(:v) do
|
|
43
|
+
AWS.stub!
|
|
44
|
+
v = Automan::Beanstalk::Version.new
|
|
45
|
+
v.logger = Logger.new('/dev/null')
|
|
46
|
+
v
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it 'ignores AWS::ElasticBeanstalk::Errors::SourceBundleDeletionFailure' do
|
|
50
|
+
eb = double(:eb)
|
|
51
|
+
eb.stub(:delete_application_version).and_raise(AWS::ElasticBeanstalk::Errors::SourceBundleDeletionFailure)
|
|
52
|
+
v.eb = eb
|
|
53
|
+
expect {
|
|
54
|
+
v.delete_by_label('foo')
|
|
55
|
+
}.not_to raise_error
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
end
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
require 'automan'
|
|
2
|
+
|
|
3
|
+
describe Automan::Cloudformation::Launcher do
|
|
4
|
+
it { should respond_to :name }
|
|
5
|
+
it { should respond_to :template }
|
|
6
|
+
it { should respond_to :parameters }
|
|
7
|
+
it { should respond_to :disable_rollback }
|
|
8
|
+
it { should respond_to :enable_iam }
|
|
9
|
+
it { should respond_to :enable_update }
|
|
10
|
+
it { should respond_to :parse_template_parameters }
|
|
11
|
+
it { should respond_to :launch_or_update }
|
|
12
|
+
it { should respond_to :launch }
|
|
13
|
+
it { should respond_to :update }
|
|
14
|
+
it { should respond_to :read_manifest }
|
|
15
|
+
it { should respond_to :wait_for_completion }
|
|
16
|
+
it { should respond_to :stack_launch_complete? }
|
|
17
|
+
it { should respond_to :stack_update_complete? }
|
|
18
|
+
|
|
19
|
+
describe '#read_manifest' do
|
|
20
|
+
subject(:s) do
|
|
21
|
+
AWS.stub!
|
|
22
|
+
s = Automan::Cloudformation::Launcher.new
|
|
23
|
+
s
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
it 'raises MissingManifestError if manifest does not exist' do
|
|
27
|
+
s.stub(:manifest_exists?).and_return(false)
|
|
28
|
+
expect {
|
|
29
|
+
s.read_manifest
|
|
30
|
+
}.to raise_error(Automan::Cloudformation::MissingManifestError)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it 'merges manifest contents into parameters hash' do
|
|
34
|
+
s.stub(:manifest_exists?).and_return(true)
|
|
35
|
+
s.stub(:s3_read).and_return('{"foo": "bar", "big": "poppa"}')
|
|
36
|
+
s.parameters = {'foo'=> 'baz', 'fo'=> 'shizzle'}
|
|
37
|
+
s.read_manifest
|
|
38
|
+
s.parameters.should eq({'foo' => 'bar', 'fo' => 'shizzle', 'big' => 'poppa'})
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
describe '#parse_template_parameters' do
|
|
43
|
+
subject(:s) do
|
|
44
|
+
AWS.stub!
|
|
45
|
+
s = Automan::Cloudformation::Launcher.new
|
|
46
|
+
s.cfn = AWS::CloudFormation.new
|
|
47
|
+
s.cfn.client.stub_for :validate_template
|
|
48
|
+
s
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
it "returns a hash" do
|
|
52
|
+
s.template = File.join(File.dirname(__FILE__), 'templates', 'worker_role.json')
|
|
53
|
+
result = s.parse_template_parameters
|
|
54
|
+
result.class.should eq(Hash)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
describe '#template_handle' do
|
|
60
|
+
subject(:s) do
|
|
61
|
+
AWS.stub!
|
|
62
|
+
s = Automan::Cloudformation::Launcher.new
|
|
63
|
+
s
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
it "returns an s3 key if it is an s3 path" do
|
|
67
|
+
s.template_handle("s3://foo/bar/baz").should be_a AWS::S3::S3Object
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it "returns a string if it is a local file" do
|
|
71
|
+
s.template_handle(__FILE__).should be_a String
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
describe '#validate_parameters' do
|
|
76
|
+
subject(:s) do
|
|
77
|
+
AWS.stub!
|
|
78
|
+
s = Automan::Cloudformation::Launcher.new
|
|
79
|
+
s.logger = Logger.new('/dev/null')
|
|
80
|
+
s.parameters = {}
|
|
81
|
+
s
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
it "raises error if no parameters were specified" do
|
|
85
|
+
s.parameters = nil
|
|
86
|
+
expect {
|
|
87
|
+
s.validate_parameters
|
|
88
|
+
}.to raise_error Automan::Cloudformation::MissingParametersError
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
it "raises error if the template doesn't validate" do
|
|
92
|
+
Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return({code: 'foo', message: 'bar'})
|
|
93
|
+
expect {
|
|
94
|
+
s.validate_parameters
|
|
95
|
+
}.to raise_error Automan::Cloudformation::BadTemplateError
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
it "raises error if a required parameter isn't present" do
|
|
99
|
+
Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo'}])
|
|
100
|
+
expect {
|
|
101
|
+
s.validate_parameters
|
|
102
|
+
}.to raise_error Automan::Cloudformation::MissingParametersError
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it "raises no error if all required parameters are present" do
|
|
106
|
+
Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo'}])
|
|
107
|
+
s.parameters = {'foo' => 'bar'}
|
|
108
|
+
expect {
|
|
109
|
+
s.validate_parameters
|
|
110
|
+
}.not_to raise_error
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
it "raises no error if there are no required parameters" do
|
|
114
|
+
Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo', default_value: 'bar'}])
|
|
115
|
+
expect {
|
|
116
|
+
s.validate_parameters
|
|
117
|
+
}.not_to raise_error
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
describe '#update' do
|
|
122
|
+
subject(:s) do
|
|
123
|
+
AWS.stub!
|
|
124
|
+
s = Automan::Cloudformation::Launcher.new
|
|
125
|
+
s.logger = Logger.new('/dev/null')
|
|
126
|
+
s.stub(:template_handle).and_return('foo')
|
|
127
|
+
s
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
it "ignores AWS::CloudFormation::Errors::ValidationError when no updates are to be performed" do
|
|
131
|
+
s.stub(:update_stack).and_raise AWS::CloudFormation::Errors::ValidationError.new("No updates are to be performed.")
|
|
132
|
+
expect {
|
|
133
|
+
s.update
|
|
134
|
+
}.to_not raise_error
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
it 're-raises any other ValidationError' do
|
|
138
|
+
s.stub(:update_stack).and_raise AWS::CloudFormation::Errors::ValidationError.new("foo")
|
|
139
|
+
expect {
|
|
140
|
+
s.update
|
|
141
|
+
}.to raise_error AWS::CloudFormation::Errors::ValidationError
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
describe '#launch_or_update' do
|
|
146
|
+
subject(:s) do
|
|
147
|
+
AWS.stub!
|
|
148
|
+
s = Automan::Cloudformation::Launcher.new
|
|
149
|
+
s.logger = Logger.new('/dev/null')
|
|
150
|
+
s.stub(:validate_parameters)
|
|
151
|
+
s
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
it "launches a new stack if it does not exist" do
|
|
155
|
+
s.stub(:stack_exists?).and_return(false)
|
|
156
|
+
s.should_receive(:launch)
|
|
157
|
+
s.launch_or_update
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
it "updates an existing stack if update is enabled" do
|
|
161
|
+
s.stub(:stack_exists?).and_return(true)
|
|
162
|
+
s.stub(:enable_update).and_return(true)
|
|
163
|
+
s.should_receive(:update)
|
|
164
|
+
s.launch_or_update
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
it "raises an error when updating an existing stack w/o update enabled" do
|
|
168
|
+
s.stub(:stack_exists?).and_return(true)
|
|
169
|
+
s.stub(:enable_update).and_return(false)
|
|
170
|
+
expect {
|
|
171
|
+
s.launch_or_update
|
|
172
|
+
}.to raise_error Automan::Cloudformation::StackExistsError
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
describe '#stack_launch_complete?' do
|
|
177
|
+
subject(:s) do
|
|
178
|
+
AWS.stub!
|
|
179
|
+
s = Automan::Cloudformation::Launcher.new
|
|
180
|
+
s.logger = Logger.new('/dev/null')
|
|
181
|
+
s
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
it 'returns true when the stack completes' do
|
|
185
|
+
s.stub(:stack_status).and_return('CREATE_COMPLETE')
|
|
186
|
+
s.stack_launch_complete?.should be_true
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
it 'raises an error when the stack fails' do
|
|
190
|
+
s.stub(:stack_status).and_return('CREATE_FAILED')
|
|
191
|
+
expect {
|
|
192
|
+
s.stack_launch_complete?
|
|
193
|
+
}.to raise_error Automan::Cloudformation::StackCreationError
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
rollback_states = %w[ROLLBACK_COMPLETE ROLLBACK_FAILED ROLLBACK_IN_PROGRESS]
|
|
197
|
+
rollback_states.each do |state|
|
|
198
|
+
it "raises an error when the stack goes into #{state} state" do
|
|
199
|
+
s.stub(:stack_status).and_return(state)
|
|
200
|
+
expect {
|
|
201
|
+
s.stack_launch_complete?
|
|
202
|
+
}.to raise_error Automan::Cloudformation::StackCreationError
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
it 'returns false for any other state' do
|
|
207
|
+
s.stub(:stack_status).and_return('foo')
|
|
208
|
+
s.stack_launch_complete?.should be_false
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
describe '#stack_update_complete?' do
|
|
213
|
+
subject(:s) do
|
|
214
|
+
AWS.stub!
|
|
215
|
+
s = Automan::Cloudformation::Launcher.new
|
|
216
|
+
s.logger = Logger.new('/dev/null')
|
|
217
|
+
s
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
it 'returns true when the stack completes' do
|
|
221
|
+
s.stub(:stack_status).and_return('UPDATE_COMPLETE')
|
|
222
|
+
s.stack_update_complete?.should be_true
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
rollback_states = %w[UPDATE_ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_IN_PROGRESS]
|
|
226
|
+
rollback_states.each do |state|
|
|
227
|
+
it "raises an error when the stack enters #{state} state" do
|
|
228
|
+
s.stub(:stack_status).and_return(state)
|
|
229
|
+
expect {
|
|
230
|
+
s.stack_update_complete?
|
|
231
|
+
}.to raise_error Automan::Cloudformation::StackUpdateError
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
it 'returns false for any other state' do
|
|
236
|
+
s.stub(:stack_status).and_return('foo')
|
|
237
|
+
s.stack_update_complete?.should be_false
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require 'automan'
|
|
2
|
+
|
|
3
|
+
describe Automan::Cloudformation::Replacer do
|
|
4
|
+
it { should respond_to :name }
|
|
5
|
+
it { should respond_to :replace_instances }
|
|
6
|
+
it { should respond_to :ok_to_replace_instances? }
|
|
7
|
+
it { should respond_to :stack_exists? }
|
|
8
|
+
|
|
9
|
+
describe '#ok_to_replace_instances?' do
|
|
10
|
+
subject(:r) do
|
|
11
|
+
AWS.stub!
|
|
12
|
+
r = Automan::Cloudformation::Replacer.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
good_states = %w[UPDATE_COMPLETE]
|
|
16
|
+
|
|
17
|
+
bad_states =
|
|
18
|
+
%w[
|
|
19
|
+
CREATE_FAILED ROLLBACK_IN_PROGRESS ROLLBACK_FAILED
|
|
20
|
+
ROLLBACK_COMPLETE DELETE_IN_PROGRESS DELETE_FAILED
|
|
21
|
+
DELETE_COMPLETE UPDATE_ROLLBACK_IN_PROGRESS
|
|
22
|
+
UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
|
|
23
|
+
UPDATE_ROLLBACK_COMPLETE
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
wait_states =
|
|
27
|
+
%w[CREATE_COMPLETE CREATE_IN_PROGRESS
|
|
28
|
+
UPDATE_IN_PROGRESS UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
good_states.each do |state|
|
|
32
|
+
it "returns true when stack is in the right state (state: #{state})" do
|
|
33
|
+
r.ok_to_replace_instances?(state, Time.now).should be_true
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
good_states.each do |state|
|
|
38
|
+
it "returns false when stack is in the right state but has not been updated in the last 5m" do
|
|
39
|
+
last_updated = Time.now - (5 * 60 + 1)
|
|
40
|
+
r.ok_to_replace_instances?(state, last_updated).should be_false
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
bad_states.each do |state|
|
|
45
|
+
it "raises error when stack is borked (state: #{state})" do
|
|
46
|
+
expect {
|
|
47
|
+
r.ok_to_replace_instances?(state, Time.now)
|
|
48
|
+
}.to raise_error Automan::Cloudformation::StackBrokenError
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
wait_states.each do |state|
|
|
53
|
+
it "returns false when it is not yet ok to replace (state: #{state})" do
|
|
54
|
+
r.ok_to_replace_instances?(state, Time.now).should be_false
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
{
|
|
2
|
+
"AWSTemplateFormatVersion" : "2010-09-09",
|
|
3
|
+
|
|
4
|
+
"Description" : "AWS CloudFormation Sample Template WorkerRole: Create a multi-az, Auto Scaled worker that pulls command messages from a queue and execs the command. Each message contains a command/script to run, an input file location and an output location for the results. The application is Auto-Scaled based on the amount of work in the queue. **WARNING** This template creates one or more Amazon EC2 instances and an Amazon SQS queue. You will be billed for the AWS resources used if you create a stack from this template.",
|
|
5
|
+
|
|
6
|
+
"Parameters" : {
|
|
7
|
+
"InstanceType" : {
|
|
8
|
+
"Description" : "Worker EC2 instance type",
|
|
9
|
+
"Type" : "String",
|
|
10
|
+
"Default" : "m1.small",
|
|
11
|
+
"AllowedValues" : [ "t1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m2.xlarge","m2.2xlarge","m2.4xlarge","c1.medium","c1.xlarge","cc1.4xlarge","cc2.8xlarge","cg1.4xlarge"],
|
|
12
|
+
"ConstraintDescription" : "must be a valid EC2 instance type."
|
|
13
|
+
},
|
|
14
|
+
|
|
15
|
+
"KeyName" : {
|
|
16
|
+
"Description" : "The EC2 Key Pair to allow SSH access to the instances",
|
|
17
|
+
"Type" : "String"
|
|
18
|
+
},
|
|
19
|
+
|
|
20
|
+
"MinInstances" : {
|
|
21
|
+
"Description" : "The minimum number of Workers",
|
|
22
|
+
"Type" : "Number",
|
|
23
|
+
"MinValue" : "0",
|
|
24
|
+
"Default" : "0",
|
|
25
|
+
"ConstraintDescription" : "Enter a number >=0"
|
|
26
|
+
},
|
|
27
|
+
|
|
28
|
+
"MaxInstances" : {
|
|
29
|
+
"Description" : "The maximum number of Workers",
|
|
30
|
+
"Type" : "Number",
|
|
31
|
+
"MinValue" : "1",
|
|
32
|
+
"Default" : "1",
|
|
33
|
+
"ConstraintDescription" : "Enter a number >1"
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
"Mappings" : {
|
|
38
|
+
"AWSInstanceType2Arch" : {
|
|
39
|
+
"t1.micro" : { "Arch" : "64" },
|
|
40
|
+
"m1.small" : { "Arch" : "64" },
|
|
41
|
+
"m1.medium" : { "Arch" : "64" },
|
|
42
|
+
"m1.large" : { "Arch" : "64" },
|
|
43
|
+
"m1.xlarge" : { "Arch" : "64" },
|
|
44
|
+
"m2.xlarge" : { "Arch" : "64" },
|
|
45
|
+
"m2.2xlarge" : { "Arch" : "64" },
|
|
46
|
+
"m2.4xlarge" : { "Arch" : "64" },
|
|
47
|
+
"c1.medium" : { "Arch" : "64" },
|
|
48
|
+
"c1.xlarge" : { "Arch" : "64" },
|
|
49
|
+
"cc1.4xlarge" : { "Arch" : "64HVM" },
|
|
50
|
+
"cc2.8xlarge" : { "Arch" : "64HVM" },
|
|
51
|
+
"cg1.4xlarge" : { "Arch" : "64HVM" }
|
|
52
|
+
},
|
|
53
|
+
|
|
54
|
+
"AWSRegionArch2AMI" : {
|
|
55
|
+
"us-east-1" : { "32" : "ami-31814f58", "64" : "ami-1b814f72", "64HVM" : "ami-0da96764" },
|
|
56
|
+
"us-west-2" : { "32" : "ami-38fe7308", "64" : "ami-30fe7300", "64HVM" : "NOT_YET_SUPPORTED" },
|
|
57
|
+
"us-west-1" : { "32" : "ami-11d68a54", "64" : "ami-1bd68a5e", "64HVM" : "NOT_YET_SUPPORTED" },
|
|
58
|
+
"eu-west-1" : { "32" : "ami-973b06e3", "64" : "ami-953b06e1", "64HVM" : "NOT_YET_SUPPORTED" },
|
|
59
|
+
"ap-southeast-1" : { "32" : "ami-b4b0cae6", "64" : "ami-beb0caec", "64HVM" : "NOT_YET_SUPPORTED" },
|
|
60
|
+
"ap-northeast-1" : { "32" : "ami-0644f007", "64" : "ami-0a44f00b", "64HVM" : "NOT_YET_SUPPORTED" },
|
|
61
|
+
"sa-east-1" : { "32" : "ami-3e3be423", "64" : "ami-3c3be421", "64HVM" : "NOT_YET_SUPPORTED" }
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
"Resources" : {
|
|
66
|
+
|
|
67
|
+
"WorkerUser" : {
|
|
68
|
+
"Type" : "AWS::IAM::User",
|
|
69
|
+
"Properties" : {
|
|
70
|
+
"Path": "/",
|
|
71
|
+
"Policies": [{
|
|
72
|
+
"PolicyName": "root",
|
|
73
|
+
"PolicyDocument": {
|
|
74
|
+
"Version": "2012-10-17",
|
|
75
|
+
"Statement":[{
|
|
76
|
+
"Effect": "Allow",
|
|
77
|
+
"Action": [
|
|
78
|
+
"cloudformation:DescribeStackResource",
|
|
79
|
+
"sqs:ReceiveMessage",
|
|
80
|
+
"sqs:DeleteMessage",
|
|
81
|
+
"sns:Publish"
|
|
82
|
+
],
|
|
83
|
+
"Resource": "*"
|
|
84
|
+
}]
|
|
85
|
+
}
|
|
86
|
+
}]
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
|
|
90
|
+
"WorkerKeys" : {
|
|
91
|
+
"Type" : "AWS::IAM::AccessKey",
|
|
92
|
+
"Properties" : {
|
|
93
|
+
"UserName" : {"Ref": "WorkerUser"}
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
|
|
97
|
+
"InputQueue" : {
|
|
98
|
+
"Type" : "AWS::SQS::Queue"
|
|
99
|
+
},
|
|
100
|
+
|
|
101
|
+
"InputQueuePolicy" : {
|
|
102
|
+
"Type" : "AWS::SQS::QueuePolicy",
|
|
103
|
+
"DependsOn" : "LaunchConfig",
|
|
104
|
+
"Properties" : {
|
|
105
|
+
"Queues" : [ { "Ref" : "InputQueue" } ],
|
|
106
|
+
"PolicyDocument": {
|
|
107
|
+
"Version": "2012-10-17",
|
|
108
|
+
"Id": "ReadFromQueuePolicy",
|
|
109
|
+
"Statement" : [ {
|
|
110
|
+
"Sid": "ConsumeMessages",
|
|
111
|
+
"Effect": "Allow",
|
|
112
|
+
"Principal" : { "AWS": {"Fn::GetAtt" : ["WorkerUser", "Arn"]} },
|
|
113
|
+
"Action": ["sqs:ReceiveMessage", "sqs:DeleteMessage"],
|
|
114
|
+
"Resource": { "Fn::GetAtt" : [ "InputQueue", "Arn" ] }
|
|
115
|
+
} ]
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
|
|
120
|
+
"InstanceSecurityGroup" : {
|
|
121
|
+
"Type" : "AWS::EC2::SecurityGroup",
|
|
122
|
+
"Properties" : {
|
|
123
|
+
"GroupDescription" : "Enable SSH access",
|
|
124
|
+
"SecurityGroupIngress" : [ { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" } ]
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
|
|
128
|
+
"LaunchConfig" : {
|
|
129
|
+
"Type" : "AWS::AutoScaling::LaunchConfiguration",
|
|
130
|
+
"Metadata" : {
|
|
131
|
+
"Comment" : "Install a simple PHP application",
|
|
132
|
+
"AWS::CloudFormation::Init" : {
|
|
133
|
+
"configSets" : {
|
|
134
|
+
"ALL" : ["XML", "Time", "LWP", "AmazonLibraries", "WorkerRole"]
|
|
135
|
+
},
|
|
136
|
+
"XML" : {
|
|
137
|
+
"packages" : {
|
|
138
|
+
"yum" : {
|
|
139
|
+
"perl-XML-Simple" : []
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
"Time" : {
|
|
144
|
+
"packages" : {
|
|
145
|
+
"yum" : {
|
|
146
|
+
"perl-LWP-Protocol-https" : []
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
"LWP" : {
|
|
151
|
+
"packages" : {
|
|
152
|
+
"yum" : {
|
|
153
|
+
"perl-Time-HiRes" : []
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
"AmazonLibraries" : {
|
|
158
|
+
"sources" : {
|
|
159
|
+
"/home/ec2-user/sqs" : "http://s3.amazonaws.com/awscode/amazon-queue/2009-02-01/perl/library/amazon-queue-2009-02-01-perl-library.zip"
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
"WorkerRole" : {
|
|
163
|
+
"files" : {
|
|
164
|
+
"/etc/cron.d/worker.cron" : {
|
|
165
|
+
"content" : "*/1 * * * * ec2-user /home/ec2-user/worker.pl &> /home/ec2-user/worker.log\n",
|
|
166
|
+
"mode" : "000644",
|
|
167
|
+
"owner" : "root",
|
|
168
|
+
"group" : "root"
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
"/home/ec2-user/worker.pl" : {
|
|
172
|
+
"content" : { "Fn::Join" : ["", [
|
|
173
|
+
"#!/usr/bin/perl -w\n",
|
|
174
|
+
"#\n",
|
|
175
|
+
"use strict;\n",
|
|
176
|
+
"use Carp qw( croak );\n",
|
|
177
|
+
"use lib qw(/home/ec2-user/sqs/amazon-queue-2009-02-01-perl-library/src); \n",
|
|
178
|
+
"use LWP::Simple qw( getstore );\n",
|
|
179
|
+
"\n",
|
|
180
|
+
"my $AWS_ACCESS_KEY_ID = \"", { "Ref" : "WorkerKeys" }, "\";\n",
|
|
181
|
+
"my $AWS_SECRET_ACCESS_KEY = \"", { "Fn::GetAtt": ["WorkerKeys", "SecretAccessKey"]}, "\";\n",
|
|
182
|
+
"my $QUEUE_NAME = \"", { "Ref" : "InputQueue" }, "\";\n",
|
|
183
|
+
"my $COMMAND_FILE = \"/home/ec2-user/command\";\n",
|
|
184
|
+
"\n",
|
|
185
|
+
"eval {\n",
|
|
186
|
+
"\n",
|
|
187
|
+
" use Amazon::SQS::Client; \n",
|
|
188
|
+
" my $service = Amazon::SQS::Client->new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY);\n",
|
|
189
|
+
" \n",
|
|
190
|
+
" my $response = $service->receiveMessage({QueueUrl=>$QUEUE_NAME, MaxNumberOfMessages=>1});\n",
|
|
191
|
+
" if ($response->isSetReceiveMessageResult) {\n",
|
|
192
|
+
" my $result = $response->getReceiveMessageResult();\n",
|
|
193
|
+
" if ($result->isSetMessage) {\n",
|
|
194
|
+
" my $messageList = $response->getReceiveMessageResult()->getMessage();\n",
|
|
195
|
+
" foreach(@$messageList) {\n",
|
|
196
|
+
" my $message = $_;\n",
|
|
197
|
+
" my $messageHandle = 0;\n",
|
|
198
|
+
" if ($message->isSetReceiptHandle()) {\n",
|
|
199
|
+
" $messageHandle = $message->getReceiptHandle();\n",
|
|
200
|
+
" } else {\n",
|
|
201
|
+
" croak \"Couldn't get message Id from message\";\n",
|
|
202
|
+
" }\n",
|
|
203
|
+
" if ($message->isSetBody()) {\n",
|
|
204
|
+
" my %parameters = split(/[=;]/, $message->getBody());\n",
|
|
205
|
+
" if (defined($parameters{\"Input\"}) && defined($parameters{\"Output\"}) && defined($parameters{\"Command\"})) {\n",
|
|
206
|
+
" getstore($parameters{\"Command\"}, $COMMAND_FILE);\n",
|
|
207
|
+
" chmod(0755, $COMMAND_FILE);\n",
|
|
208
|
+
" my $command = $COMMAND_FILE . \" \" . $parameters{\"Input\"} . \" \" . $parameters{\"Output\"};\n",
|
|
209
|
+
" my $result = `$command`;\n",
|
|
210
|
+
" print \"Result = \" . $result . \"\\n\";\n",
|
|
211
|
+
" } else {\n",
|
|
212
|
+
" croak \"Invalid message\";\n",
|
|
213
|
+
" }\n",
|
|
214
|
+
" } else {\n",
|
|
215
|
+
" croak \"Couldn't get message body from message\";\n",
|
|
216
|
+
" }\n",
|
|
217
|
+
" my $response = $service->deleteMessage({QueueUrl=>$QUEUE_NAME, ReceiptHandle=>$messageHandle});\n",
|
|
218
|
+
" }\n",
|
|
219
|
+
" } else {\n",
|
|
220
|
+
" printf \"Empty Poll\\n\";\n",
|
|
221
|
+
" }\n",
|
|
222
|
+
" } else {\n",
|
|
223
|
+
" croak \"Call failed\";\n",
|
|
224
|
+
" }\n",
|
|
225
|
+
"}; \n",
|
|
226
|
+
"\n",
|
|
227
|
+
"my $ex = $@;\n",
|
|
228
|
+
"if ($ex) {\n",
|
|
229
|
+
" require Amazon::SQS::Exception;\n",
|
|
230
|
+
" if (ref $ex eq \"Amazon::SQS::Exception\") {\n",
|
|
231
|
+
" print(\"Caught Exception: \" . $ex->getMessage() . \"\\n\");\n",
|
|
232
|
+
" } else {\n",
|
|
233
|
+
" croak $@;\n",
|
|
234
|
+
" }\n",
|
|
235
|
+
"}\n"
|
|
236
|
+
]]},
|
|
237
|
+
"mode" : "000755",
|
|
238
|
+
"owner" : "ec2-user",
|
|
239
|
+
"group" : "ec2-user"
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
},
|
|
245
|
+
"Properties" : {
|
|
246
|
+
"KeyName" : { "Ref" : "KeyName" },
|
|
247
|
+
"SpotPrice" : "0.05",
|
|
248
|
+
"ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" },
|
|
249
|
+
{ "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" },
|
|
250
|
+
"Arch" ] } ] },
|
|
251
|
+
"SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ],
|
|
252
|
+
"InstanceType" : { "Ref" : "InstanceType" },
|
|
253
|
+
"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
|
|
254
|
+
"#!/bin/bash\n",
|
|
255
|
+
"yum update -y aws-cfn-bootstrap\n",
|
|
256
|
+
"# Install the Worker application\n",
|
|
257
|
+
"/opt/aws/bin/cfn-init ",
|
|
258
|
+
" --stack ", { "Ref" : "AWS::StackName" },
|
|
259
|
+
" --resource LaunchConfig ",
|
|
260
|
+
" --configset ALL",
|
|
261
|
+
" --access-key ", { "Ref" : "WorkerKeys" },
|
|
262
|
+
" --secret-key ", {"Fn::GetAtt": ["WorkerKeys", "SecretAccessKey"]},
|
|
263
|
+
" --region ", { "Ref" : "AWS::Region" }, "\n"
|
|
264
|
+
]]}}
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
|
|
268
|
+
"WorkerGroup" : {
|
|
269
|
+
"Type" : "AWS::AutoScaling::AutoScalingGroup",
|
|
270
|
+
"Properties" : {
|
|
271
|
+
"AvailabilityZones" : { "Fn::GetAZs" : ""},
|
|
272
|
+
"LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
|
|
273
|
+
"MinSize" : { "Ref" : "MinInstances" },
|
|
274
|
+
"MaxSize" : { "Ref" : "MaxInstances" }
|
|
275
|
+
}
|
|
276
|
+
},
|
|
277
|
+
|
|
278
|
+
"WorkerScaleUpPolicy" : {
|
|
279
|
+
"Type" : "AWS::AutoScaling::ScalingPolicy",
|
|
280
|
+
"Properties" : {
|
|
281
|
+
"AdjustmentType" : "ChangeInCapacity",
|
|
282
|
+
"AutoScalingGroupName" : { "Ref" : "WorkerGroup" },
|
|
283
|
+
"Cooldown" : "60",
|
|
284
|
+
"ScalingAdjustment" : "1"
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
|
|
288
|
+
"WorkerScaleDownPolicy" : {
|
|
289
|
+
"Type" : "AWS::AutoScaling::ScalingPolicy",
|
|
290
|
+
"Properties" : {
|
|
291
|
+
"AdjustmentType" : "ChangeInCapacity",
|
|
292
|
+
"AutoScalingGroupName" : { "Ref" : "WorkerGroup" },
|
|
293
|
+
"Cooldown" : "60",
|
|
294
|
+
"ScalingAdjustment" : "-1"
|
|
295
|
+
}
|
|
296
|
+
},
|
|
297
|
+
|
|
298
|
+
"TooManyMessagesAlarm": {
|
|
299
|
+
"Type": "AWS::CloudWatch::Alarm",
|
|
300
|
+
"Properties": {
|
|
301
|
+
"AlarmDescription": "Scale-Up if queue depth grows beyond 10 messages",
|
|
302
|
+
"Namespace": "AWS/SQS",
|
|
303
|
+
"MetricName": "ApproximateNumberOfMessagesVisible",
|
|
304
|
+
"Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }],
|
|
305
|
+
"Statistic": "Sum",
|
|
306
|
+
"Period": "60",
|
|
307
|
+
"EvaluationPeriods": "3",
|
|
308
|
+
"Threshold": "1",
|
|
309
|
+
"ComparisonOperator": "GreaterThanThreshold",
|
|
310
|
+
"AlarmActions": [ { "Ref": "WorkerScaleUpPolicy" } ]
|
|
311
|
+
}
|
|
312
|
+
},
|
|
313
|
+
|
|
314
|
+
"NotEnoughMessagesAlarm": {
|
|
315
|
+
"Type": "AWS::CloudWatch::Alarm",
|
|
316
|
+
"Properties": {
|
|
317
|
+
"AlarmDescription": "Scale-down if there are too many empty polls, indicating there is not enough work",
|
|
318
|
+
"Namespace": "AWS/SQS",
|
|
319
|
+
"MetricName": "NumberOfEmptyReceives",
|
|
320
|
+
"Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }],
|
|
321
|
+
"Statistic": "Sum",
|
|
322
|
+
"Period": "60",
|
|
323
|
+
"EvaluationPeriods": "10",
|
|
324
|
+
"Threshold": "3",
|
|
325
|
+
"ComparisonOperator": "GreaterThanThreshold",
|
|
326
|
+
"AlarmActions": [ { "Ref": "WorkerScaleDownPolicy" } ]
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
},
|
|
330
|
+
|
|
331
|
+
"Outputs" : {
|
|
332
|
+
"QueueURL" : {
|
|
333
|
+
"Description" : "URL of input queue",
|
|
334
|
+
"Value" : { "Ref" : "InputQueue" }
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|