staugaard-cloudmaster 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/VERSION.yml +4 -0
  2. data/bin/cloudmaster +45 -0
  3. data/lib/AWS/AWS.rb +3 -0
  4. data/lib/AWS/EC2.rb +14 -0
  5. data/lib/AWS/S3.rb +14 -0
  6. data/lib/AWS/SQS.rb +14 -0
  7. data/lib/AWS/SimpleDB.rb +14 -0
  8. data/lib/MockAWS/EC2.rb +119 -0
  9. data/lib/MockAWS/S3.rb +39 -0
  10. data/lib/MockAWS/SQS.rb +82 -0
  11. data/lib/MockAWS/SimpleDB.rb +46 -0
  12. data/lib/MockAWS/clock.rb +67 -0
  13. data/lib/OriginalAWS/AWS.rb +475 -0
  14. data/lib/OriginalAWS/EC2.rb +783 -0
  15. data/lib/OriginalAWS/S3.rb +559 -0
  16. data/lib/OriginalAWS/SQS.rb +159 -0
  17. data/lib/OriginalAWS/SimpleDB.rb +460 -0
  18. data/lib/RetryAWS/EC2.rb +88 -0
  19. data/lib/RetryAWS/S3.rb +77 -0
  20. data/lib/RetryAWS/SQS.rb +109 -0
  21. data/lib/RetryAWS/SimpleDB.rb +118 -0
  22. data/lib/SafeAWS/EC2.rb +63 -0
  23. data/lib/SafeAWS/S3.rb +56 -0
  24. data/lib/SafeAWS/SQS.rb +75 -0
  25. data/lib/SafeAWS/SimpleDB.rb +88 -0
  26. data/lib/aws_context.rb +165 -0
  27. data/lib/basic_configuration.rb +120 -0
  28. data/lib/clock.rb +10 -0
  29. data/lib/factory.rb +14 -0
  30. data/lib/file_logger.rb +36 -0
  31. data/lib/inifile.rb +148 -0
  32. data/lib/instance_logger.rb +25 -0
  33. data/lib/logger_factory.rb +38 -0
  34. data/lib/periodic.rb +29 -0
  35. data/lib/string_logger.rb +29 -0
  36. data/lib/sys_logger.rb +40 -0
  37. data/lib/user_data.rb +30 -0
  38. data/test/aws-config.ini +9 -0
  39. data/test/cloudmaster-tests.rb +329 -0
  40. data/test/configuration-test.rb +62 -0
  41. data/test/daytime-policy-tests.rb +47 -0
  42. data/test/enumerator-test.rb +47 -0
  43. data/test/fixed-policy-tests.rb +50 -0
  44. data/test/instance-pool-test.rb +359 -0
  45. data/test/instance-test.rb +98 -0
  46. data/test/job-policy-test.rb +95 -0
  47. data/test/manual-policy-tests.rb +63 -0
  48. data/test/named-queue-test.rb +90 -0
  49. data/test/resource-policy-tests.rb +126 -0
  50. data/test/suite +17 -0
  51. data/test/test-config.ini +47 -0
  52. metadata +111 -0
@@ -0,0 +1,329 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "app"))
2
+ require 'test/unit'
3
+ require 'stringio'
4
+ require 'rexml/document'
5
+ require 'time'
6
+ require 'MockAWS/clock'
7
+ require 'logger_factory'
8
+ require 'configuration'
9
+ require 'pool_configuration'
10
+ require 'pool_runner'
11
+ require 'aws_context'
12
+ require 'pp'
13
+
14
+ # Test the Cloudmaster class.
15
+ class CloudmasterTests < Test::Unit::TestCase
16
+ def setup
17
+ # Append output to string
18
+ LoggerFactory.setup("/tmp/test.out")
19
+ logf = LoggerFactory.create(:file)
20
+ AwsContext.setup(:mock, logf)
21
+ @ec2 = AwsContext.instance.ec2
22
+ @sqs = AwsContext.instance.sqs
23
+ Clock.reset
24
+ @ps = nil
25
+ end
26
+
27
+ def teardown
28
+ @ps.shutdown if @ps
29
+ end
30
+
31
+ def configure(opts)
32
+ config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
33
+ Cloudmaster::Configuration.setup_config_files(config_files)
34
+ @cfg = Cloudmaster::Configuration.new([], opts)
35
+ end
36
+
37
+ def startup
38
+ @ps = Cloudmaster::PoolRunner.new(@cfg)
39
+ end
40
+
41
+ # Send message to primes work queue.
42
+ def send_work
43
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"
44
+ body = "1|3649"
45
+ @sqs.send_message(url, body)
46
+ end
47
+
48
+ # Send message to fib work queue.
49
+ def send_fib_work
50
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test"
51
+ body = "test"
52
+ @sqs.send_message(url, body)
53
+ end
54
+
55
+ # Receive a message from primes work queue.
56
+ def rec_message
57
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"
58
+ @sqs.receive_messages(url)
59
+ end
60
+
61
+ # Delete a message from primes work queue, given its id.
62
+ def del_message(receipt_handle)
63
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"
64
+ @sqs.delete_message(url, receipt_handle)
65
+ end
66
+
67
+ # Both receive and delete message from primes work queue.
68
+ def consume_message
69
+ msg = rec_message
70
+ del_message(msg.first[:receipt_handle])
71
+ end
72
+
73
+ # Send a message on the primes status queue.
74
+ def send_status_message(inst, load = 1)
75
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test"
76
+ body = YAML.dump({ :type => 'status',
77
+ :instance_id => inst,
78
+ :state => 'active',
79
+ :load_estimate => load,
80
+ :timestamp => Clock.now})
81
+ @sqs.send_message(url, body)
82
+ end
83
+
84
+ # Send a message on the primes status queue.
85
+ def send_status_message_lifeguard(inst, load = 1)
86
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test"
87
+ doc = REXML::Document.new
88
+ is = doc.add_element 'InstanceStatus'
89
+ iid = is.add_element 'InstanceId'
90
+ iid.add_text inst
91
+ st = is.add_element 'State'
92
+ st.add_text 'busy'
93
+ li = is.add_element 'LastInterval'
94
+ li.add_text 'PT60S'
95
+ ts = is.add_element 'Timestamp'
96
+ # ts.add_text(Time.now.xmlschema)
97
+ ts.add_text(Clock.now.time.to_s)
98
+ body = String.new
99
+ doc.write(body, 2)
100
+ @sqs.send_message(url, body)
101
+ end
102
+
103
+ # Send a message on the fib status queue.
104
+ def send_fib_status_message(inst, load = 1)
105
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test"
106
+ body = YAML.dump({ :type => 'status',
107
+ :instance_id => inst,
108
+ :state => 'active',
109
+ :load_estimate => load,
110
+ :timestamp => Clock.now})
111
+ @sqs.send_message(url, body)
112
+ end
113
+
114
+ # Send a log message on the primes status queue.
115
+ def send_log_message(msg)
116
+ url = "http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test"
117
+ msg = { :type => 'log',
118
+ :instance_id => 'iid-fake',
119
+ :message => msg,
120
+ :timestamp => Clock.now}
121
+ @sqs.send_message(url, YAML.dump(msg))
122
+ end
123
+
124
+ # verify nothing created when queue empty
125
+ def test_idle
126
+ configure([:primes]); startup
127
+ @ps.run(Clock.at(65))
128
+ assert_equal(0, @ec2.count)
129
+ end
130
+
131
+ # verify create when queue occupied
132
+ def test_create_by_queue
133
+ configure([:primes]); startup
134
+ send_work
135
+ @ps.run(Clock.at(65))
136
+ assert_equal(1, @ec2.count)
137
+ end
138
+
139
+ # verify audit picks up existing instances
140
+ def test_discover
141
+ configure([:primes]); startup
142
+ ami_id = @ec2.valid_ami_id
143
+ opts = {}
144
+ @ec2.run_instances(ami_id, 1, 1, opts)
145
+ @ps.run(Clock.at(65))
146
+ assert_equal(1, @ec2.count)
147
+ end
148
+
149
+ # verify audit detects missng instances
150
+ def test_missing
151
+ configure([:primes]); startup
152
+ @ps = Cloudmaster::PoolRunner.new(@cfg)
153
+ end
154
+
155
+ # verify minimum insances
156
+ def test_minimum
157
+ configure([:primes])
158
+ @cfg.pools[0][:minimum_number_of_instances] = 1
159
+ startup
160
+ @ps.run(Clock.at(65))
161
+ assert_equal(1, @ec2.count)
162
+ end
163
+
164
+ # verify creation to max
165
+ def test_maximum
166
+ configure([:primes]); startup
167
+ send_work
168
+ @ps.run(Clock.at(65))
169
+ assert_equal(1, @ec2.count)
170
+ # only three -- no increase
171
+ send_work; send_work
172
+ @ps.run(Clock.at(125))
173
+ assert_equal(1, @ec2.count)
174
+ # now four -- increase
175
+ send_work
176
+ @ps.run(Clock.at(185))
177
+ assert_equal(2, @ec2.count)
178
+ # still four queued -- ensure total does not increase
179
+ @ps.run(Clock.at(245))
180
+ assert_equal(2, @ec2.count)
181
+ end
182
+
183
+ # verify handling log message
184
+ def test_log_message
185
+ configure([:primes]); startup
186
+ send_log_message('this is a test')
187
+ @ps.run(Clock.at(5))
188
+ logger = @ps.pool_managers[0].logger
189
+ assert_match("0 primes iid-fake this is a test", logger.string)
190
+ end
191
+
192
+ # verify handling status message
193
+ def test_status_message
194
+ configure([:primes]); startup
195
+ send_work
196
+ @ps.run(Clock.at(65))
197
+ @ps.run(Clock.at(125))
198
+ assert_equal(:startup, @ps.pool_managers[0].instances.first.state)
199
+ send_status_message('i-1')
200
+ @ps.run(Clock.at(185))
201
+ assert_equal(:active, @ps.pool_managers[0].instances.first.state)
202
+ end
203
+
204
+ # verify update public dns
205
+ def test_public_dns
206
+ configure([:primes]); startup
207
+ send_work
208
+ @ps.run(Clock.at(65))
209
+ assert_equal(nil, @ps.pool_managers[0].instances.first.public_dns)
210
+ id = @ec2.first_id
211
+ assert_equal("", @ec2.get_public_dns(id))
212
+ @ec2.set_public_dns(id, "dns-xx")
213
+ @ps.run(Clock.at(75))
214
+ assert_equal("dns-xx", @ec2.get_public_dns(id))
215
+ assert_equal("dns-xx", @ps.pool_managers[0].instances.first.public_dns)
216
+ end
217
+
218
+ # verify job policy increase
219
+ def test_job_increase
220
+ configure([:primes]); startup
221
+ send_work; send_work
222
+ @ps.run(Clock.at(65))
223
+ assert_equal(1, @ec2.count)
224
+ # let it run longer -- shouled create another
225
+ send_work; send_work
226
+ @ps.run(Clock.at(125))
227
+ assert_equal(2, @ec2.count)
228
+ end
229
+
230
+ # verify job policy no change
231
+ def test_job_same
232
+ configure([:primes]); startup
233
+ send_work
234
+ @ps.run(Clock.at(65))
235
+ assert_equal(1, @ec2.count)
236
+ msg = rec_message
237
+ del_message(msg.first[:receipt_handle])
238
+ # let it run longer -- shouled not create another
239
+ @ps.run(Clock.at(125))
240
+ assert_equal(1, @ec2.count)
241
+ end
242
+
243
+ # verify job policy decrease
244
+ def test_job_decrease
245
+ configure([:primes]); startup
246
+ 4.times { send_work }
247
+ @ps.run(Clock.at(65))
248
+ assert_equal(2, @ec2.count)
249
+ 4.times { consume_message }
250
+ send_status_message('i-1', 0)
251
+ send_status_message('i-2', 0)
252
+ # run to next policy evaluation
253
+ @ps.run(Clock.at(400))
254
+ assert_equal(0, @ec2.count)
255
+ end
256
+
257
+ # verify resource policy increase (many cases)
258
+ def test_resource_increase
259
+ configure([:fib]); startup
260
+ send_fib_work
261
+ @ps.run(Clock.at(65))
262
+ assert_equal(1, @ec2.count)
263
+ end
264
+
265
+ # verify resource policy no change
266
+ def test_resource_same
267
+ configure([:fib]); startup
268
+ send_fib_work
269
+ @ps.run(Clock.at(65))
270
+ assert_equal(1, @ec2.count)
271
+ @ps.pool_managers[0].instances.first.state = :active
272
+ @ps.pool_managers[0].instances.first.load_estimate = 0.5
273
+ @ps.run(Clock.at(400))
274
+ assert_equal(1, @ec2.count)
275
+ end
276
+
277
+ # verify resource policy decrease
278
+ def test_resource_decrease
279
+ configure([:fib]); startup
280
+ send_fib_work
281
+ @ps.run(Clock.at(65))
282
+ assert_equal(1, @ec2.count)
283
+ send_fib_status_message('i-1', 0.1)
284
+ @ps.run(Clock.at(400))
285
+ assert_equal(1, @ec2.count)
286
+ send_fib_status_message('i-1', 0)
287
+ @ps.run(Clock.at(465))
288
+ assert_equal(0, @ec2.count)
289
+ end
290
+
291
+ # verify write active set
292
+ def test_active_set
293
+ configure([:primes]); startup
294
+ # let it run
295
+ # read from S3 and test if active-set is there and has right value
296
+ send_work
297
+ @ps.run(Clock.at(125))
298
+ val = AwsContext.instance.s3.get_object('chayden', 'active-set/primes-instances-test')
299
+ assert_equal("--- []\n\n", val)
300
+ end
301
+
302
+ # verify hung instances cleaned up
303
+ def test_hang
304
+ configure([:primes]); startup
305
+ #let it run for a long time, until min lifetime and hang time
306
+ # see that it stops the instance
307
+ send_work
308
+ @ps.run(Clock.at(65))
309
+ assert_equal(1, @ec2.count)
310
+ @ps.run(Clock.at(700))
311
+ assert_equal(0, @ec2.count)
312
+ end
313
+
314
+ # verify handling status message
315
+ def test_status_message_lifeguard
316
+ configure([:primes]);
317
+ @cfg.pools[0][:status_parser] = 'lifeguard'
318
+ startup
319
+ send_work
320
+ @ps.run(Clock.at(65))
321
+ @ps.run(Clock.at(125))
322
+ assert_equal(:startup, @ps.pool_managers[0].instances.first.state)
323
+ send_status_message_lifeguard('i-1')
324
+ @ps.run(Clock.at(185))
325
+ assert_equal(:active, @ps.pool_managers[0].instances.first.state)
326
+ end
327
+
328
+ end
329
+
@@ -0,0 +1,62 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "app"))
2
+ require 'pp'
3
+ require 'test/unit'
4
+ require 'MockAWS/clock'
5
+ require 'logger_factory'
6
+ require 'configuration'
7
+ require 'pool_configuration'
8
+ require 'aws_context'
9
+ require 'reporter'
10
+
11
+ # Test the ConfigInfo class.
12
+ # This also test InifileConfig
13
+ class ConfigInfoTests < Test::Unit::TestCase
14
+ def setup
15
+ LoggerFactory.setup("/tmp/test.out")
16
+ logf = LoggerFactory.create(:file)
17
+ @sqs = AwsContext.setup(:mock, logf).sqs
18
+ config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
19
+ Cloudmaster::Configuration.setup_config_files(config_files)
20
+ tc = Cloudmaster::Configuration.new([], [:primes])
21
+ cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
22
+ reporter = Cloudmaster::Reporter.setup(cfg[:name], logf)
23
+ cfg[:ami_id] = "ami-08856161"
24
+ @cfg = cfg
25
+ end
26
+
27
+ def test_get
28
+ assert_equal(nil, @cfg.get(:xxx))
29
+ assert_equal(:primes, @cfg.get(:name))
30
+ assert_equal(20, @cfg.get(:receive_count).to_i)
31
+ end
32
+
33
+ def test_fetch
34
+ assert_equal(:primes, @cfg[:name])
35
+ assert_equal(20, @cfg[:receive_count].to_i)
36
+ assert_raise(RuntimeError) do
37
+ @cfg[:xxx]
38
+ end
39
+ end
40
+
41
+ def test_store
42
+ assert_equal(:primes, @cfg[:name])
43
+ @cfg[:name] = "new-name"
44
+ assert_equal('new-name', @cfg[:name])
45
+ assert_equal(20, @cfg[:receive_count].to_i)
46
+ @cfg[:receive_count] = 30
47
+ assert_equal(30, @cfg[:receive_count])
48
+ assert_raise(RuntimeError) do
49
+ @cfg[:xxx]
50
+ end
51
+ @cfg[:xxx] = 5
52
+ assert_equal(5, @cfg[:xxx])
53
+ end
54
+
55
+ def test_groups
56
+ assert_equal('["a", "b"]', @cfg[:security_groups])
57
+ end
58
+
59
+ def test_user_data
60
+ assert_equal(1234, @cfg[:user_data][:newkey])
61
+ end
62
+ end
@@ -0,0 +1,47 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "app"))
2
+ require 'test/unit'
3
+ require 'logger_factory'
4
+ require 'MockAWS/clock'
5
+ require 'configuration'
6
+ require 'pool_configuration'
7
+ require 'aws_context'
8
+ require 'pp'
9
+
10
+ # Test the DaytimePolicy class.
11
+ # This class is an example of adding an additional policy to the system.
12
+ class DaytimePolicyTests < Test::Unit::TestCase
13
+ def setup
14
+ LoggerFactory.setup("/tmp/test.out")
15
+ logf = LoggerFactory.create(:file)
16
+ config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
17
+ Cloudmaster::Configuration.setup_config_files(config_files)
18
+ tc = Cloudmaster::Configuration.new([], [:daytime])
19
+ cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
20
+ reporter = Cloudmaster::Reporter.setup(cfg[:name], logf)
21
+ cfg[:ami_id] = "ami-08856161"
22
+ @cfg = cfg
23
+ AwsContext.setup(:mock, logf)
24
+ @pool = Cloudmaster::InstancePool.new(reporter, @cfg)
25
+ @policy = Cloudmaster::PolicyFactory.create(@cfg[:policy], reporter, @cfg, @pool)
26
+ Clock.reset
27
+ end
28
+
29
+ def test_daytime_load
30
+ assert_equal(Cloudmaster::PolicyDaytime, @policy.class)
31
+ end
32
+
33
+ def test_nighttime
34
+ assert_equal(0, @policy.adjust)
35
+ end
36
+
37
+ def test_daytime_additional
38
+ Clock.set(11 * 3600)
39
+ assert_equal(1, @policy.adjust)
40
+ end
41
+
42
+ def test_daytime_max
43
+ @pool.start_n_instances(3)
44
+ Clock.set(11 * 3600)
45
+ assert_equal(0, @policy.adjust)
46
+ end
47
+ end
@@ -0,0 +1,47 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "app"))
2
+ require 'test/unit'
3
+ require 'MockAWS/clock'
4
+ require 'logger_factory'
5
+ require 'aws_context'
6
+ require 'ec2_instance_enumerator'
7
+ require 'ec2_image_enumerator'
8
+ require 'pp'
9
+
10
+ # tests the EC2ImageEnumerator and the C2InstanceEnumerators.
11
+ class EnumeratorTests < Test::Unit::TestCase
12
+ def setup
13
+ LoggerFactory.setup("/tmp/test.out")
14
+ logf = LoggerFactory.create(:file)
15
+ @ec2 = AwsContext.setup(:mock, logf).ec2
16
+ @image_enum = Cloudmaster::EC2ImageEnumerator.new
17
+ @inst_enum = Cloudmaster::EC2InstanceEnumerator.new
18
+ end
19
+
20
+ def test_each
21
+ images = []
22
+ @image_enum.each {|i| images << i}
23
+ assert_equal(3, images.size)
24
+ end
25
+
26
+ def test_find_image_id_by_name
27
+ id = @image_enum.find_image_id_by_name("ami-primes-test")
28
+ assert_equal('ami-08856161', id)
29
+ assert_raise(RuntimeError) do
30
+ id = @image_enum.find_image_id_by_name("xxx")
31
+ end
32
+ assert_raise(RuntimeError) do
33
+ id = @image_enum.find_image_id_by_name("test")
34
+ end
35
+ end
36
+
37
+ def test_instance_each
38
+ inst = []
39
+ @inst_enum.each {|i| inst << i}
40
+ assert_equal(0, inst.size)
41
+ id1 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
42
+ @inst_enum = Cloudmaster::EC2InstanceEnumerator.new
43
+ inst = []
44
+ @inst_enum.each {|i| inst << i}
45
+ assert_equal(1, inst.size)
46
+ end
47
+ end
@@ -0,0 +1,50 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "app"))
2
+ require 'test/unit'
3
+ require 'MockAWS/clock'
4
+ require 'logger_factory'
5
+ require 'configuration'
6
+ require 'pool_configuration'
7
+ require 'aws_context'
8
+ require 'pp'
9
+
10
+ # Test that the standard policy classes can be created, and have the
11
+ # right class names. This tests the dynamic loading of the policy code.
12
+ # It tests the job and resource policies, as well as the fixed policy.
13
+ class FixedPolicyTests < Test::Unit::TestCase
14
+ def startup(pool)
15
+ LoggerFactory.setup("/tmp/test.out")
16
+ logf = LoggerFactory.create(:file)
17
+ config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
18
+ Cloudmaster::Configuration.setup_config_files(config_files)
19
+ tc = Cloudmaster::Configuration.new([], [pool])
20
+ cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
21
+ reporter = Cloudmaster::Reporter.setup(cfg[:name], logf)
22
+ cfg[:ami_id] = "ami-08856161"
23
+ @cfg = cfg
24
+ AwsContext.setup(:mock, logf)
25
+ @pool = Cloudmaster::InstancePool.new(reporter, @cfg)
26
+ @policy = Cloudmaster::PolicyFactory.create(@cfg[:policy], reporter, @cfg, @pool)
27
+ end
28
+
29
+ def test_primes
30
+ startup(:primes)
31
+ assert_equal(Cloudmaster::PolicyJob, @policy.class)
32
+ end
33
+
34
+ def test_fib
35
+ startup(:fib)
36
+ assert_equal(Cloudmaster::PolicyResource, @policy.class)
37
+ end
38
+
39
+ def test_fixed_policy
40
+ startup(:fixed)
41
+ assert_equal(Cloudmaster::PolicyFixed, @policy.class)
42
+ end
43
+
44
+ def test_bad_policy
45
+ assert_raise(LoadError) do
46
+ startup(:bad)
47
+ Cloudmaster::PolicyDefault
48
+ end
49
+ end
50
+ end