automan 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rvmrc +1 -0
  4. data/.travis.yml +5 -0
  5. data/Gemfile +4 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +29 -0
  8. data/Rakefile +6 -0
  9. data/automan.gemspec +30 -0
  10. data/bin/baker +4 -0
  11. data/bin/mover +4 -0
  12. data/bin/scanner +4 -0
  13. data/bin/snapper +4 -0
  14. data/bin/stacker +4 -0
  15. data/bin/stalker +4 -0
  16. data/lib/automan.rb +27 -0
  17. data/lib/automan/base.rb +64 -0
  18. data/lib/automan/beanstalk/application.rb +74 -0
  19. data/lib/automan/beanstalk/configuration.rb +137 -0
  20. data/lib/automan/beanstalk/deployer.rb +193 -0
  21. data/lib/automan/beanstalk/errors.rb +22 -0
  22. data/lib/automan/beanstalk/package.rb +39 -0
  23. data/lib/automan/beanstalk/router.rb +102 -0
  24. data/lib/automan/beanstalk/terminator.rb +60 -0
  25. data/lib/automan/beanstalk/uploader.rb +58 -0
  26. data/lib/automan/beanstalk/version.rb +100 -0
  27. data/lib/automan/chef/uploader.rb +30 -0
  28. data/lib/automan/cli/baker.rb +63 -0
  29. data/lib/automan/cli/base.rb +14 -0
  30. data/lib/automan/cli/mover.rb +47 -0
  31. data/lib/automan/cli/scanner.rb +24 -0
  32. data/lib/automan/cli/snapper.rb +78 -0
  33. data/lib/automan/cli/stacker.rb +106 -0
  34. data/lib/automan/cli/stalker.rb +279 -0
  35. data/lib/automan/cloudformation/errors.rb +40 -0
  36. data/lib/automan/cloudformation/launcher.rb +196 -0
  37. data/lib/automan/cloudformation/replacer.rb +102 -0
  38. data/lib/automan/cloudformation/terminator.rb +61 -0
  39. data/lib/automan/cloudformation/uploader.rb +57 -0
  40. data/lib/automan/ec2/errors.rb +4 -0
  41. data/lib/automan/ec2/image.rb +137 -0
  42. data/lib/automan/ec2/instance.rb +83 -0
  43. data/lib/automan/mixins/aws_caller.rb +115 -0
  44. data/lib/automan/mixins/utils.rb +18 -0
  45. data/lib/automan/rds/errors.rb +7 -0
  46. data/lib/automan/rds/snapshot.rb +244 -0
  47. data/lib/automan/s3/downloader.rb +25 -0
  48. data/lib/automan/s3/uploader.rb +20 -0
  49. data/lib/automan/version.rb +3 -0
  50. data/lib/automan/wait_rescuer.rb +17 -0
  51. data/spec/beanstalk/application_spec.rb +49 -0
  52. data/spec/beanstalk/configuration_spec.rb +98 -0
  53. data/spec/beanstalk/deployer_spec.rb +162 -0
  54. data/spec/beanstalk/package_spec.rb +9 -0
  55. data/spec/beanstalk/router_spec.rb +65 -0
  56. data/spec/beanstalk/terminator_spec.rb +67 -0
  57. data/spec/beanstalk/uploader_spec.rb +53 -0
  58. data/spec/beanstalk/version_spec.rb +60 -0
  59. data/spec/chef/uploader_spec.rb +9 -0
  60. data/spec/cloudformation/launcher_spec.rb +240 -0
  61. data/spec/cloudformation/replacer_spec.rb +58 -0
  62. data/spec/cloudformation/templates/worker_role.json +337 -0
  63. data/spec/cloudformation/terminator_spec.rb +63 -0
  64. data/spec/cloudformation/uploader_spec.rb +50 -0
  65. data/spec/ec2/image_spec.rb +158 -0
  66. data/spec/ec2/instance_spec.rb +57 -0
  67. data/spec/mixins/aws_caller_spec.rb +39 -0
  68. data/spec/mixins/utils_spec.rb +44 -0
  69. data/spec/rds/snapshot_spec.rb +152 -0
  70. metadata +278 -0
@@ -0,0 +1,53 @@
1
+ require 'automan'
2
+
3
+ describe Automan::Beanstalk::Uploader do
4
+ subject(:u) do
5
+ AWS.stub!
6
+ u = Automan::Beanstalk::Uploader.new
7
+ u.logger = Logger.new('/dev/null')
8
+ u.template_files = 'foo'
9
+ u.stub(:config_templates).and_return(['foo'])
10
+ u
11
+ end
12
+
13
+ it { should respond_to :template_files }
14
+ it { should respond_to :s3_path }
15
+ it { should respond_to :upload_config_templates }
16
+ it { should respond_to :config_templates_valid? }
17
+
18
+ describe '#config_templates_valid?' do
19
+
20
+ it 'raises error if config templates do not exist' do
21
+ u.stub(:config_templates).and_return([])
22
+ expect {
23
+ u.config_templates_valid?
24
+ }.to raise_error(Automan::Beanstalk::NoConfigurationTemplatesError)
25
+ end
26
+
27
+ it 'returns true if config templates are valid json' do
28
+ u.stub(:read_config_template).and_return('[{}]')
29
+ u.config_templates_valid?.should be_true
30
+ end
31
+
32
+ it 'returns false if config templates are invalid json' do
33
+ u.stub(:read_config_template).and_return('@#$%#')
34
+ u.config_templates_valid?.should be_false
35
+ end
36
+ end
37
+
38
+ describe '#upload_config_templates' do
39
+ it 'raises error if any config template fails validation' do
40
+ u.stub(:config_templates_valid?).and_return(false)
41
+ expect {
42
+ u.upload_config_templates
43
+ }.to raise_error(Automan::Beanstalk::InvalidConfigurationTemplateError)
44
+ end
45
+
46
+ it 'uploads files if they are valid' do
47
+ u.stub(:config_templates).and_return(%w[a b c])
48
+ u.stub(:config_templates_valid?).and_return(true)
49
+ u.should_receive(:upload_file).exactly(3).times
50
+ u.upload_config_templates
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,60 @@
1
+ require "automan"
2
+
3
+ describe Automan::Beanstalk::Version do
4
+
5
+ it { should respond_to :application }
6
+ it { should respond_to :label }
7
+ it { should respond_to :create }
8
+ it { should respond_to :delete }
9
+ it { should respond_to :cull_versions }
10
+ it { should respond_to :delete_by_label }
11
+
12
+ describe '#exists?' do
13
+ subject(:v) do
14
+ AWS.stub!
15
+ v = Automan::Beanstalk::Version.new
16
+ v.eb = AWS::ElasticBeanstalk::Client.new
17
+ v.application = 'foo'
18
+ v.label = 'v4'
19
+ v.log_aws_calls = false
20
+ v.logger = Logger.new('/dev/null')
21
+ v
22
+ end
23
+
24
+ it "is true when version exists in response" do
25
+ resp = v.eb.stub_for :describe_application_versions
26
+ resp.data[:application_versions] = [
27
+ {application_name: v.application, version_label: v.label}
28
+ ]
29
+ v.exists?.should be_true
30
+ end
31
+
32
+ it "is false when version is not in response" do
33
+ resp = v.eb.stub_for :describe_application_versions
34
+ resp.data[:application_versions] = [
35
+ {application_name: '', version_label: ''}
36
+ ]
37
+ v.exists?.should be_false
38
+ end
39
+ end
40
+
41
+ describe '#delete_by_label' do
42
+ subject(:v) do
43
+ AWS.stub!
44
+ v = Automan::Beanstalk::Version.new
45
+ v.logger = Logger.new('/dev/null')
46
+ v
47
+ end
48
+
49
+ it 'ignores AWS::ElasticBeanstalk::Errors::SourceBundleDeletionFailure' do
50
+ eb = double(:eb)
51
+ eb.stub(:delete_application_version).and_raise(AWS::ElasticBeanstalk::Errors::SourceBundleDeletionFailure)
52
+ v.eb = eb
53
+ expect {
54
+ v.delete_by_label('foo')
55
+ }.not_to raise_error
56
+ end
57
+
58
+ end
59
+
60
+ end
@@ -0,0 +1,9 @@
1
+ require "automan"
2
+
3
+ describe Automan::Chef::Uploader do
4
+ it { should respond_to :repopath }
5
+ it { should respond_to :s3path }
6
+ it { should respond_to :chefver }
7
+ it { should respond_to :tempdir }
8
+ it { should respond_to :upload }
9
+ end
@@ -0,0 +1,240 @@
1
+ require 'automan'
2
+
3
+ describe Automan::Cloudformation::Launcher do
4
+ it { should respond_to :name }
5
+ it { should respond_to :template }
6
+ it { should respond_to :parameters }
7
+ it { should respond_to :disable_rollback }
8
+ it { should respond_to :enable_iam }
9
+ it { should respond_to :enable_update }
10
+ it { should respond_to :parse_template_parameters }
11
+ it { should respond_to :launch_or_update }
12
+ it { should respond_to :launch }
13
+ it { should respond_to :update }
14
+ it { should respond_to :read_manifest }
15
+ it { should respond_to :wait_for_completion }
16
+ it { should respond_to :stack_launch_complete? }
17
+ it { should respond_to :stack_update_complete? }
18
+
19
+ describe '#read_manifest' do
20
+ subject(:s) do
21
+ AWS.stub!
22
+ s = Automan::Cloudformation::Launcher.new
23
+ s
24
+ end
25
+
26
+ it 'raises MissingManifestError if manifest does not exist' do
27
+ s.stub(:manifest_exists?).and_return(false)
28
+ expect {
29
+ s.read_manifest
30
+ }.to raise_error(Automan::Cloudformation::MissingManifestError)
31
+ end
32
+
33
+ it 'merges manifest contents into parameters hash' do
34
+ s.stub(:manifest_exists?).and_return(true)
35
+ s.stub(:s3_read).and_return('{"foo": "bar", "big": "poppa"}')
36
+ s.parameters = {'foo'=> 'baz', 'fo'=> 'shizzle'}
37
+ s.read_manifest
38
+ s.parameters.should eq({'foo' => 'bar', 'fo' => 'shizzle', 'big' => 'poppa'})
39
+ end
40
+ end
41
+
42
+ describe '#parse_template_parameters' do
43
+ subject(:s) do
44
+ AWS.stub!
45
+ s = Automan::Cloudformation::Launcher.new
46
+ s.cfn = AWS::CloudFormation.new
47
+ s.cfn.client.stub_for :validate_template
48
+ s
49
+ end
50
+
51
+ it "returns a hash" do
52
+ s.template = File.join(File.dirname(__FILE__), 'templates', 'worker_role.json')
53
+ result = s.parse_template_parameters
54
+ result.class.should eq(Hash)
55
+ end
56
+
57
+ end
58
+
59
+ describe '#template_handle' do
60
+ subject(:s) do
61
+ AWS.stub!
62
+ s = Automan::Cloudformation::Launcher.new
63
+ s
64
+ end
65
+
66
+ it "returns an s3 key if it is an s3 path" do
67
+ s.template_handle("s3://foo/bar/baz").should be_a AWS::S3::S3Object
68
+ end
69
+
70
+ it "returns a string if it is a local file" do
71
+ s.template_handle(__FILE__).should be_a String
72
+ end
73
+ end
74
+
75
+ describe '#validate_parameters' do
76
+ subject(:s) do
77
+ AWS.stub!
78
+ s = Automan::Cloudformation::Launcher.new
79
+ s.logger = Logger.new('/dev/null')
80
+ s.parameters = {}
81
+ s
82
+ end
83
+
84
+ it "raises error if no parameters were specified" do
85
+ s.parameters = nil
86
+ expect {
87
+ s.validate_parameters
88
+ }.to raise_error Automan::Cloudformation::MissingParametersError
89
+ end
90
+
91
+ it "raises error if the template doesn't validate" do
92
+ Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return({code: 'foo', message: 'bar'})
93
+ expect {
94
+ s.validate_parameters
95
+ }.to raise_error Automan::Cloudformation::BadTemplateError
96
+ end
97
+
98
+ it "raises error if a required parameter isn't present" do
99
+ Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo'}])
100
+ expect {
101
+ s.validate_parameters
102
+ }.to raise_error Automan::Cloudformation::MissingParametersError
103
+ end
104
+
105
+ it "raises no error if all required parameters are present" do
106
+ Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo'}])
107
+ s.parameters = {'foo' => 'bar'}
108
+ expect {
109
+ s.validate_parameters
110
+ }.not_to raise_error
111
+ end
112
+
113
+ it "raises no error if there are no required parameters" do
114
+ Automan::Cloudformation::Launcher.any_instance.stub(:parse_template_parameters).and_return(parameters: [{parameter_key: 'foo', default_value: 'bar'}])
115
+ expect {
116
+ s.validate_parameters
117
+ }.not_to raise_error
118
+ end
119
+ end
120
+
121
+ describe '#update' do
122
+ subject(:s) do
123
+ AWS.stub!
124
+ s = Automan::Cloudformation::Launcher.new
125
+ s.logger = Logger.new('/dev/null')
126
+ s.stub(:template_handle).and_return('foo')
127
+ s
128
+ end
129
+
130
+ it "ignores AWS::CloudFormation::Errors::ValidationError when no updates are to be performed" do
131
+ s.stub(:update_stack).and_raise AWS::CloudFormation::Errors::ValidationError.new("No updates are to be performed.")
132
+ expect {
133
+ s.update
134
+ }.to_not raise_error
135
+ end
136
+
137
+ it 're-raises any other ValidationError' do
138
+ s.stub(:update_stack).and_raise AWS::CloudFormation::Errors::ValidationError.new("foo")
139
+ expect {
140
+ s.update
141
+ }.to raise_error AWS::CloudFormation::Errors::ValidationError
142
+ end
143
+ end
144
+
145
+ describe '#launch_or_update' do
146
+ subject(:s) do
147
+ AWS.stub!
148
+ s = Automan::Cloudformation::Launcher.new
149
+ s.logger = Logger.new('/dev/null')
150
+ s.stub(:validate_parameters)
151
+ s
152
+ end
153
+
154
+ it "launches a new stack if it does not exist" do
155
+ s.stub(:stack_exists?).and_return(false)
156
+ s.should_receive(:launch)
157
+ s.launch_or_update
158
+ end
159
+
160
+ it "updates an existing stack if update is enabled" do
161
+ s.stub(:stack_exists?).and_return(true)
162
+ s.stub(:enable_update).and_return(true)
163
+ s.should_receive(:update)
164
+ s.launch_or_update
165
+ end
166
+
167
+ it "raises an error when updating an existing stack w/o update enabled" do
168
+ s.stub(:stack_exists?).and_return(true)
169
+ s.stub(:enable_update).and_return(false)
170
+ expect {
171
+ s.launch_or_update
172
+ }.to raise_error Automan::Cloudformation::StackExistsError
173
+ end
174
+ end
175
+
176
+ describe '#stack_launch_complete?' do
177
+ subject(:s) do
178
+ AWS.stub!
179
+ s = Automan::Cloudformation::Launcher.new
180
+ s.logger = Logger.new('/dev/null')
181
+ s
182
+ end
183
+
184
+ it 'returns true when the stack completes' do
185
+ s.stub(:stack_status).and_return('CREATE_COMPLETE')
186
+ s.stack_launch_complete?.should be_true
187
+ end
188
+
189
+ it 'raises an error when the stack fails' do
190
+ s.stub(:stack_status).and_return('CREATE_FAILED')
191
+ expect {
192
+ s.stack_launch_complete?
193
+ }.to raise_error Automan::Cloudformation::StackCreationError
194
+ end
195
+
196
+ rollback_states = %w[ROLLBACK_COMPLETE ROLLBACK_FAILED ROLLBACK_IN_PROGRESS]
197
+ rollback_states.each do |state|
198
+ it "raises an error when the stack goes into #{state} state" do
199
+ s.stub(:stack_status).and_return(state)
200
+ expect {
201
+ s.stack_launch_complete?
202
+ }.to raise_error Automan::Cloudformation::StackCreationError
203
+ end
204
+ end
205
+
206
+ it 'returns false for any other state' do
207
+ s.stub(:stack_status).and_return('foo')
208
+ s.stack_launch_complete?.should be_false
209
+ end
210
+ end
211
+
212
+ describe '#stack_update_complete?' do
213
+ subject(:s) do
214
+ AWS.stub!
215
+ s = Automan::Cloudformation::Launcher.new
216
+ s.logger = Logger.new('/dev/null')
217
+ s
218
+ end
219
+
220
+ it 'returns true when the stack completes' do
221
+ s.stub(:stack_status).and_return('UPDATE_COMPLETE')
222
+ s.stack_update_complete?.should be_true
223
+ end
224
+
225
+ rollback_states = %w[UPDATE_ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_IN_PROGRESS]
226
+ rollback_states.each do |state|
227
+ it "raises an error when the stack enters #{state} state" do
228
+ s.stub(:stack_status).and_return(state)
229
+ expect {
230
+ s.stack_update_complete?
231
+ }.to raise_error Automan::Cloudformation::StackUpdateError
232
+ end
233
+ end
234
+
235
+ it 'returns false for any other state' do
236
+ s.stub(:stack_status).and_return('foo')
237
+ s.stack_update_complete?.should be_false
238
+ end
239
+ end
240
+ end
@@ -0,0 +1,58 @@
1
+ require 'automan'
2
+
3
+ describe Automan::Cloudformation::Replacer do
4
+ it { should respond_to :name }
5
+ it { should respond_to :replace_instances }
6
+ it { should respond_to :ok_to_replace_instances? }
7
+ it { should respond_to :stack_exists? }
8
+
9
+ describe '#ok_to_replace_instances?' do
10
+ subject(:r) do
11
+ AWS.stub!
12
+ r = Automan::Cloudformation::Replacer.new
13
+ end
14
+
15
+ good_states = %w[UPDATE_COMPLETE]
16
+
17
+ bad_states =
18
+ %w[
19
+ CREATE_FAILED ROLLBACK_IN_PROGRESS ROLLBACK_FAILED
20
+ ROLLBACK_COMPLETE DELETE_IN_PROGRESS DELETE_FAILED
21
+ DELETE_COMPLETE UPDATE_ROLLBACK_IN_PROGRESS
22
+ UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
23
+ UPDATE_ROLLBACK_COMPLETE
24
+ ]
25
+
26
+ wait_states =
27
+ %w[CREATE_COMPLETE CREATE_IN_PROGRESS
28
+ UPDATE_IN_PROGRESS UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
29
+ ]
30
+
31
+ good_states.each do |state|
32
+ it "returns true when stack is in the right state (state: #{state})" do
33
+ r.ok_to_replace_instances?(state, Time.now).should be_true
34
+ end
35
+ end
36
+
37
+ good_states.each do |state|
38
+ it "returns false when stack is in the right state but has not been updated in the last 5m" do
39
+ last_updated = Time.now - (5 * 60 + 1)
40
+ r.ok_to_replace_instances?(state, last_updated).should be_false
41
+ end
42
+ end
43
+
44
+ bad_states.each do |state|
45
+ it "raises error when stack is borked (state: #{state})" do
46
+ expect {
47
+ r.ok_to_replace_instances?(state, Time.now)
48
+ }.to raise_error Automan::Cloudformation::StackBrokenError
49
+ end
50
+ end
51
+
52
+ wait_states.each do |state|
53
+ it "returns false when it is not yet ok to replace (state: #{state})" do
54
+ r.ok_to_replace_instances?(state, Time.now).should be_false
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,337 @@
1
+ {
2
+ "AWSTemplateFormatVersion" : "2010-09-09",
3
+
4
+ "Description" : "AWS CloudFormation Sample Template WorkerRole: Create a multi-az, Auto Scaled worker that pulls command messages from a queue and execs the command. Each message contains a command/script to run, an input file location and an output location for the results. The application is Auto-Scaled based on the amount of work in the queue. **WARNING** This template creates one or more Amazon EC2 instances and an Amazon SQS queue. You will be billed for the AWS resources used if you create a stack from this template.",
5
+
6
+ "Parameters" : {
7
+ "InstanceType" : {
8
+ "Description" : "Worker EC2 instance type",
9
+ "Type" : "String",
10
+ "Default" : "m1.small",
11
+ "AllowedValues" : [ "t1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m2.xlarge","m2.2xlarge","m2.4xlarge","c1.medium","c1.xlarge","cc1.4xlarge","cc2.8xlarge","cg1.4xlarge"],
12
+ "ConstraintDescription" : "must be a valid EC2 instance type."
13
+ },
14
+
15
+ "KeyName" : {
16
+ "Description" : "The EC2 Key Pair to allow SSH access to the instances",
17
+ "Type" : "String"
18
+ },
19
+
20
+ "MinInstances" : {
21
+ "Description" : "The minimum number of Workers",
22
+ "Type" : "Number",
23
+ "MinValue" : "0",
24
+ "Default" : "0",
25
+ "ConstraintDescription" : "Enter a number >=0"
26
+ },
27
+
28
+ "MaxInstances" : {
29
+ "Description" : "The maximum number of Workers",
30
+ "Type" : "Number",
31
+ "MinValue" : "1",
32
+ "Default" : "1",
33
+ "ConstraintDescription" : "Enter a number >1"
34
+ }
35
+ },
36
+
37
+ "Mappings" : {
38
+ "AWSInstanceType2Arch" : {
39
+ "t1.micro" : { "Arch" : "64" },
40
+ "m1.small" : { "Arch" : "64" },
41
+ "m1.medium" : { "Arch" : "64" },
42
+ "m1.large" : { "Arch" : "64" },
43
+ "m1.xlarge" : { "Arch" : "64" },
44
+ "m2.xlarge" : { "Arch" : "64" },
45
+ "m2.2xlarge" : { "Arch" : "64" },
46
+ "m2.4xlarge" : { "Arch" : "64" },
47
+ "c1.medium" : { "Arch" : "64" },
48
+ "c1.xlarge" : { "Arch" : "64" },
49
+ "cc1.4xlarge" : { "Arch" : "64HVM" },
50
+ "cc2.8xlarge" : { "Arch" : "64HVM" },
51
+ "cg1.4xlarge" : { "Arch" : "64HVM" }
52
+ },
53
+
54
+ "AWSRegionArch2AMI" : {
55
+ "us-east-1" : { "32" : "ami-31814f58", "64" : "ami-1b814f72", "64HVM" : "ami-0da96764" },
56
+ "us-west-2" : { "32" : "ami-38fe7308", "64" : "ami-30fe7300", "64HVM" : "NOT_YET_SUPPORTED" },
57
+ "us-west-1" : { "32" : "ami-11d68a54", "64" : "ami-1bd68a5e", "64HVM" : "NOT_YET_SUPPORTED" },
58
+ "eu-west-1" : { "32" : "ami-973b06e3", "64" : "ami-953b06e1", "64HVM" : "NOT_YET_SUPPORTED" },
59
+ "ap-southeast-1" : { "32" : "ami-b4b0cae6", "64" : "ami-beb0caec", "64HVM" : "NOT_YET_SUPPORTED" },
60
+ "ap-northeast-1" : { "32" : "ami-0644f007", "64" : "ami-0a44f00b", "64HVM" : "NOT_YET_SUPPORTED" },
61
+ "sa-east-1" : { "32" : "ami-3e3be423", "64" : "ami-3c3be421", "64HVM" : "NOT_YET_SUPPORTED" }
62
+ }
63
+ },
64
+
65
+ "Resources" : {
66
+
67
+ "WorkerUser" : {
68
+ "Type" : "AWS::IAM::User",
69
+ "Properties" : {
70
+ "Path": "/",
71
+ "Policies": [{
72
+ "PolicyName": "root",
73
+ "PolicyDocument": {
74
+ "Version": "2012-10-17",
75
+ "Statement":[{
76
+ "Effect": "Allow",
77
+ "Action": [
78
+ "cloudformation:DescribeStackResource",
79
+ "sqs:ReceiveMessage",
80
+ "sqs:DeleteMessage",
81
+ "sns:Publish"
82
+ ],
83
+ "Resource": "*"
84
+ }]
85
+ }
86
+ }]
87
+ }
88
+ },
89
+
90
+ "WorkerKeys" : {
91
+ "Type" : "AWS::IAM::AccessKey",
92
+ "Properties" : {
93
+ "UserName" : {"Ref": "WorkerUser"}
94
+ }
95
+ },
96
+
97
+ "InputQueue" : {
98
+ "Type" : "AWS::SQS::Queue"
99
+ },
100
+
101
+ "InputQueuePolicy" : {
102
+ "Type" : "AWS::SQS::QueuePolicy",
103
+ "DependsOn" : "LaunchConfig",
104
+ "Properties" : {
105
+ "Queues" : [ { "Ref" : "InputQueue" } ],
106
+ "PolicyDocument": {
107
+ "Version": "2012-10-17",
108
+ "Id": "ReadFromQueuePolicy",
109
+ "Statement" : [ {
110
+ "Sid": "ConsumeMessages",
111
+ "Effect": "Allow",
112
+ "Principal" : { "AWS": {"Fn::GetAtt" : ["WorkerUser", "Arn"]} },
113
+ "Action": ["sqs:ReceiveMessage", "sqs:DeleteMessage"],
114
+ "Resource": { "Fn::GetAtt" : [ "InputQueue", "Arn" ] }
115
+ } ]
116
+ }
117
+ }
118
+ },
119
+
120
+ "InstanceSecurityGroup" : {
121
+ "Type" : "AWS::EC2::SecurityGroup",
122
+ "Properties" : {
123
+ "GroupDescription" : "Enable SSH access",
124
+ "SecurityGroupIngress" : [ { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" } ]
125
+ }
126
+ },
127
+
128
+ "LaunchConfig" : {
129
+ "Type" : "AWS::AutoScaling::LaunchConfiguration",
130
+ "Metadata" : {
131
+ "Comment" : "Install a simple PHP application",
132
+ "AWS::CloudFormation::Init" : {
133
+ "configSets" : {
134
+ "ALL" : ["XML", "Time", "LWP", "AmazonLibraries", "WorkerRole"]
135
+ },
136
+ "XML" : {
137
+ "packages" : {
138
+ "yum" : {
139
+ "perl-XML-Simple" : []
140
+ }
141
+ }
142
+ },
143
+ "Time" : {
144
+ "packages" : {
145
+ "yum" : {
146
+ "perl-LWP-Protocol-https" : []
147
+ }
148
+ }
149
+ },
150
+ "LWP" : {
151
+ "packages" : {
152
+ "yum" : {
153
+ "perl-Time-HiRes" : []
154
+ }
155
+ }
156
+ },
157
+ "AmazonLibraries" : {
158
+ "sources" : {
159
+ "/home/ec2-user/sqs" : "http://s3.amazonaws.com/awscode/amazon-queue/2009-02-01/perl/library/amazon-queue-2009-02-01-perl-library.zip"
160
+ }
161
+ },
162
+ "WorkerRole" : {
163
+ "files" : {
164
+ "/etc/cron.d/worker.cron" : {
165
+ "content" : "*/1 * * * * ec2-user /home/ec2-user/worker.pl &> /home/ec2-user/worker.log\n",
166
+ "mode" : "000644",
167
+ "owner" : "root",
168
+ "group" : "root"
169
+ },
170
+
171
+ "/home/ec2-user/worker.pl" : {
172
+ "content" : { "Fn::Join" : ["", [
173
+ "#!/usr/bin/perl -w\n",
174
+ "#\n",
175
+ "use strict;\n",
176
+ "use Carp qw( croak );\n",
177
+ "use lib qw(/home/ec2-user/sqs/amazon-queue-2009-02-01-perl-library/src); \n",
178
+ "use LWP::Simple qw( getstore );\n",
179
+ "\n",
180
+ "my $AWS_ACCESS_KEY_ID = \"", { "Ref" : "WorkerKeys" }, "\";\n",
181
+ "my $AWS_SECRET_ACCESS_KEY = \"", { "Fn::GetAtt": ["WorkerKeys", "SecretAccessKey"]}, "\";\n",
182
+ "my $QUEUE_NAME = \"", { "Ref" : "InputQueue" }, "\";\n",
183
+ "my $COMMAND_FILE = \"/home/ec2-user/command\";\n",
184
+ "\n",
185
+ "eval {\n",
186
+ "\n",
187
+ " use Amazon::SQS::Client; \n",
188
+ " my $service = Amazon::SQS::Client->new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY);\n",
189
+ " \n",
190
+ " my $response = $service->receiveMessage({QueueUrl=>$QUEUE_NAME, MaxNumberOfMessages=>1});\n",
191
+ " if ($response->isSetReceiveMessageResult) {\n",
192
+ " my $result = $response->getReceiveMessageResult();\n",
193
+ " if ($result->isSetMessage) {\n",
194
+ " my $messageList = $response->getReceiveMessageResult()->getMessage();\n",
195
+ " foreach(@$messageList) {\n",
196
+ " my $message = $_;\n",
197
+ " my $messageHandle = 0;\n",
198
+ " if ($message->isSetReceiptHandle()) {\n",
199
+ " $messageHandle = $message->getReceiptHandle();\n",
200
+ " } else {\n",
201
+ " croak \"Couldn't get message Id from message\";\n",
202
+ " }\n",
203
+ " if ($message->isSetBody()) {\n",
204
+ " my %parameters = split(/[=;]/, $message->getBody());\n",
205
+ " if (defined($parameters{\"Input\"}) && defined($parameters{\"Output\"}) && defined($parameters{\"Command\"})) {\n",
206
+ " getstore($parameters{\"Command\"}, $COMMAND_FILE);\n",
207
+ " chmod(0755, $COMMAND_FILE);\n",
208
+ " my $command = $COMMAND_FILE . \" \" . $parameters{\"Input\"} . \" \" . $parameters{\"Output\"};\n",
209
+ " my $result = `$command`;\n",
210
+ " print \"Result = \" . $result . \"\\n\";\n",
211
+ " } else {\n",
212
+ " croak \"Invalid message\";\n",
213
+ " }\n",
214
+ " } else {\n",
215
+ " croak \"Couldn't get message body from message\";\n",
216
+ " }\n",
217
+ " my $response = $service->deleteMessage({QueueUrl=>$QUEUE_NAME, ReceiptHandle=>$messageHandle});\n",
218
+ " }\n",
219
+ " } else {\n",
220
+ " printf \"Empty Poll\\n\";\n",
221
+ " }\n",
222
+ " } else {\n",
223
+ " croak \"Call failed\";\n",
224
+ " }\n",
225
+ "}; \n",
226
+ "\n",
227
+ "my $ex = $@;\n",
228
+ "if ($ex) {\n",
229
+ " require Amazon::SQS::Exception;\n",
230
+ " if (ref $ex eq \"Amazon::SQS::Exception\") {\n",
231
+ " print(\"Caught Exception: \" . $ex->getMessage() . \"\\n\");\n",
232
+ " } else {\n",
233
+ " croak $@;\n",
234
+ " }\n",
235
+ "}\n"
236
+ ]]},
237
+ "mode" : "000755",
238
+ "owner" : "ec2-user",
239
+ "group" : "ec2-user"
240
+ }
241
+ }
242
+ }
243
+ }
244
+ },
245
+ "Properties" : {
246
+ "KeyName" : { "Ref" : "KeyName" },
247
+ "SpotPrice" : "0.05",
248
+ "ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" },
249
+ { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" },
250
+ "Arch" ] } ] },
251
+ "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ],
252
+ "InstanceType" : { "Ref" : "InstanceType" },
253
+ "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
254
+ "#!/bin/bash\n",
255
+ "yum update -y aws-cfn-bootstrap\n",
256
+ "# Install the Worker application\n",
257
+ "/opt/aws/bin/cfn-init ",
258
+ " --stack ", { "Ref" : "AWS::StackName" },
259
+ " --resource LaunchConfig ",
260
+ " --configset ALL",
261
+ " --access-key ", { "Ref" : "WorkerKeys" },
262
+ " --secret-key ", {"Fn::GetAtt": ["WorkerKeys", "SecretAccessKey"]},
263
+ " --region ", { "Ref" : "AWS::Region" }, "\n"
264
+ ]]}}
265
+ }
266
+ },
267
+
268
+ "WorkerGroup" : {
269
+ "Type" : "AWS::AutoScaling::AutoScalingGroup",
270
+ "Properties" : {
271
+ "AvailabilityZones" : { "Fn::GetAZs" : ""},
272
+ "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
273
+ "MinSize" : { "Ref" : "MinInstances" },
274
+ "MaxSize" : { "Ref" : "MaxInstances" }
275
+ }
276
+ },
277
+
278
+ "WorkerScaleUpPolicy" : {
279
+ "Type" : "AWS::AutoScaling::ScalingPolicy",
280
+ "Properties" : {
281
+ "AdjustmentType" : "ChangeInCapacity",
282
+ "AutoScalingGroupName" : { "Ref" : "WorkerGroup" },
283
+ "Cooldown" : "60",
284
+ "ScalingAdjustment" : "1"
285
+ }
286
+ },
287
+
288
+ "WorkerScaleDownPolicy" : {
289
+ "Type" : "AWS::AutoScaling::ScalingPolicy",
290
+ "Properties" : {
291
+ "AdjustmentType" : "ChangeInCapacity",
292
+ "AutoScalingGroupName" : { "Ref" : "WorkerGroup" },
293
+ "Cooldown" : "60",
294
+ "ScalingAdjustment" : "-1"
295
+ }
296
+ },
297
+
298
+ "TooManyMessagesAlarm": {
299
+ "Type": "AWS::CloudWatch::Alarm",
300
+ "Properties": {
301
+ "AlarmDescription": "Scale-Up if queue depth grows beyond 10 messages",
302
+ "Namespace": "AWS/SQS",
303
+ "MetricName": "ApproximateNumberOfMessagesVisible",
304
+ "Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }],
305
+ "Statistic": "Sum",
306
+ "Period": "60",
307
+ "EvaluationPeriods": "3",
308
+ "Threshold": "1",
309
+ "ComparisonOperator": "GreaterThanThreshold",
310
+ "AlarmActions": [ { "Ref": "WorkerScaleUpPolicy" } ]
311
+ }
312
+ },
313
+
314
+ "NotEnoughMessagesAlarm": {
315
+ "Type": "AWS::CloudWatch::Alarm",
316
+ "Properties": {
317
+ "AlarmDescription": "Scale-down if there are too many empty polls, indicating there is not enough work",
318
+ "Namespace": "AWS/SQS",
319
+ "MetricName": "NumberOfEmptyReceives",
320
+ "Dimensions": [{ "Name": "QueueName", "Value" : { "Fn::GetAtt" : ["InputQueue", "QueueName"] } }],
321
+ "Statistic": "Sum",
322
+ "Period": "60",
323
+ "EvaluationPeriods": "10",
324
+ "Threshold": "3",
325
+ "ComparisonOperator": "GreaterThanThreshold",
326
+ "AlarmActions": [ { "Ref": "WorkerScaleDownPolicy" } ]
327
+ }
328
+ }
329
+ },
330
+
331
+ "Outputs" : {
332
+ "QueueURL" : {
333
+ "Description" : "URL of input queue",
334
+ "Value" : { "Ref" : "InputQueue" }
335
+ }
336
+ }
337
+ }