staugaard-cloudmaster 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/VERSION.yml +4 -0
  2. data/bin/cloudmaster +45 -0
  3. data/lib/AWS/AWS.rb +3 -0
  4. data/lib/AWS/EC2.rb +14 -0
  5. data/lib/AWS/S3.rb +14 -0
  6. data/lib/AWS/SQS.rb +14 -0
  7. data/lib/AWS/SimpleDB.rb +14 -0
  8. data/lib/MockAWS/EC2.rb +119 -0
  9. data/lib/MockAWS/S3.rb +39 -0
  10. data/lib/MockAWS/SQS.rb +82 -0
  11. data/lib/MockAWS/SimpleDB.rb +46 -0
  12. data/lib/MockAWS/clock.rb +67 -0
  13. data/lib/OriginalAWS/AWS.rb +475 -0
  14. data/lib/OriginalAWS/EC2.rb +783 -0
  15. data/lib/OriginalAWS/S3.rb +559 -0
  16. data/lib/OriginalAWS/SQS.rb +159 -0
  17. data/lib/OriginalAWS/SimpleDB.rb +460 -0
  18. data/lib/RetryAWS/EC2.rb +88 -0
  19. data/lib/RetryAWS/S3.rb +77 -0
  20. data/lib/RetryAWS/SQS.rb +109 -0
  21. data/lib/RetryAWS/SimpleDB.rb +118 -0
  22. data/lib/SafeAWS/EC2.rb +63 -0
  23. data/lib/SafeAWS/S3.rb +56 -0
  24. data/lib/SafeAWS/SQS.rb +75 -0
  25. data/lib/SafeAWS/SimpleDB.rb +88 -0
  26. data/lib/aws_context.rb +165 -0
  27. data/lib/basic_configuration.rb +120 -0
  28. data/lib/clock.rb +10 -0
  29. data/lib/factory.rb +14 -0
  30. data/lib/file_logger.rb +36 -0
  31. data/lib/inifile.rb +148 -0
  32. data/lib/instance_logger.rb +25 -0
  33. data/lib/logger_factory.rb +38 -0
  34. data/lib/periodic.rb +29 -0
  35. data/lib/string_logger.rb +29 -0
  36. data/lib/sys_logger.rb +40 -0
  37. data/lib/user_data.rb +30 -0
  38. data/test/aws-config.ini +9 -0
  39. data/test/cloudmaster-tests.rb +329 -0
  40. data/test/configuration-test.rb +62 -0
  41. data/test/daytime-policy-tests.rb +47 -0
  42. data/test/enumerator-test.rb +47 -0
  43. data/test/fixed-policy-tests.rb +50 -0
  44. data/test/instance-pool-test.rb +359 -0
  45. data/test/instance-test.rb +98 -0
  46. data/test/job-policy-test.rb +95 -0
  47. data/test/manual-policy-tests.rb +63 -0
  48. data/test/named-queue-test.rb +90 -0
  49. data/test/resource-policy-tests.rb +126 -0
  50. data/test/suite +17 -0
  51. data/test/test-config.ini +47 -0
  52. metadata +111 -0
@@ -0,0 +1,77 @@
1
+ require 'AWS/S3'
2
+
3
+ module RetryAWS
4
+ class S3
5
+ def initialize(access_key, secret_key)
6
+ @s3 = AWS::S3.new(access_key, secret_key)
7
+ @@log = STDOUT
8
+ @retry_limit = 16
9
+ end
10
+
11
+ def logger=(logger)
12
+ @@log = logger
13
+ end
14
+
15
+ private
16
+
17
+ def report_error(res)
18
+ @@log.puts "error #{$!}"
19
+ $@.each {|line| @@log.puts " #{line}"}
20
+ res
21
+ end
22
+
23
+ def retry?(err, retry_time)
24
+ if err.response.code >= 500 && retry_time < @retry_limit
25
+ sleep retry_time
26
+ return retry_time * 2
27
+ end
28
+ nil
29
+ end
30
+
31
+ public
32
+
33
+ # Create an S3 object.
34
+ # Catch errors, but if one occurs, return false.
35
+ def create_object(bucket_name, object_key, opts={})
36
+ retry_time = 1
37
+ begin
38
+ @s3.create_object(bucket_name, object_key, opts)
39
+ rescue AWS::ServiceError => err
40
+ retry if retry_time = retry?(err, retry_time)
41
+ report_error false
42
+ rescue
43
+ report_error false
44
+ end
45
+ end
46
+
47
+ def get_object(bucket_name, object_key, headers={})
48
+ retry_time = 1
49
+ begin
50
+ if block_given?
51
+ @s3.get_object(bucket_name, object_key, headers) do |segment|
52
+ yield(segment)
53
+ end
54
+ else
55
+ @s3.get_object(bucket_name, object_key, headers)
56
+ end
57
+ rescue AWS::ServiceError => err
58
+ retry if retry_time = retry?(err, retry_time)
59
+ report_error nil
60
+ rescue
61
+ report_error nil
62
+ end
63
+ end
64
+
65
+ def set_canned_acl(canned_acl, bucket_name, object_key='')
66
+ retry_time = 1
67
+ begin
68
+ @s3.set_canned_acl(canned_acl, bucket_name, object_key)
69
+ rescue AWS::ServiceError => err
70
+ retry if retry_time = retry?(err, retry_time)
71
+ report_error false
72
+ rescue
73
+ report_error false
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,109 @@
1
+ require 'AWS/SQS'
2
+
3
+ module RetryAWS
4
+ # Wrap SQS routines that we use.
5
+ # Catch exceptions and return something reasonable.
6
+ class SQS
7
+ def initialize(*params)
8
+ @sqs = AWS::SQS.new(*params)
9
+ @@log = STDOUT
10
+ @retry_limit = 16
11
+ end
12
+
13
+ def logger=(logger)
14
+ @@log = logger
15
+ end
16
+
17
+ private
18
+
19
+ # report error and return result
20
+ def report_error(res)
21
+ @@log.puts "error #{$!}"
22
+ $@.each {|line| @@log.puts " #{line}"}
23
+ res
24
+ end
25
+
26
+ def retry?(err, retry_time)
27
+ pp err
28
+ if err.response.code.to_i >= 500 && retry_time < @retry_limit
29
+ sleep retry_time
30
+ return retry_time * 2
31
+ end
32
+ nil
33
+ end
34
+
35
+ public
36
+
37
+ def list_queues(queue_name_prefix = nil)
38
+ retry_time = 1
39
+ begin
40
+ @sqs.list_queues(queue_name_prefix)
41
+ rescue AWS::ServiceError => err
42
+ retry if retry_time = retry?(err, retry_time)
43
+ report_error []
44
+ rescue
45
+ report_error []
46
+ end
47
+ end
48
+
49
+ def create_queue(queue_name, visibility_timeout_secs = nil)
50
+ retry_time = 1
51
+ begin
52
+ @sqs.create_queue(queue_name, visibility_timeout_secs)
53
+ rescue AWS::ServiceError => err
54
+ retry if retry_time = retry?(err, retry_time)
55
+ report_error nil
56
+ rescue
57
+ report_error nil
58
+ end
59
+ end
60
+
61
+ def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
62
+ retry_time = 1
63
+ begin
64
+ @sqs.receive_messages(queue_url, maximum, visibility_timeout)
65
+ rescue AWS::ServiceError => err
66
+ retry if retry_time = retry?(err, retry_time)
67
+ report_error []
68
+ rescue
69
+ report_error []
70
+ end
71
+ end
72
+
73
+ def send_message(queue_url, message_body, encode=false)
74
+ retry_time = 1
75
+ begin
76
+ @sqs.send_message(queue_url, message_body, encode)
77
+ rescue AWS::ServiceError => err
78
+ retry if retry_time = retry?(err, retry_time)
79
+ report_error ''
80
+ rescue
81
+ report_error ''
82
+ end
83
+ end
84
+
85
+ def delete_message(queue_url, receipt_handle)
86
+ retry_time = 1
87
+ begin
88
+ @sqs.delete_message(queue_url, receipt_handle)
89
+ rescue AWS::ServiceError => err
90
+ retry if retry_time = retry?(err, retry_time)
91
+ report_error false
92
+ rescue
93
+ report_error false
94
+ end
95
+ end
96
+
97
+ def get_queue_attributes(queue_url, attribute='All')
98
+ retry_time = 1
99
+ begin
100
+ @sqs.get_queue_attributes(queue_url, attribute)
101
+ rescue AWS::ServiceError => err
102
+ retry if retry_time = retry?(err, retry_time)
103
+ report_error({})
104
+ rescue
105
+ report_error({})
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,118 @@
1
+ require 'AWS/SimpleDB'
2
+
3
+ # RetryAWS wraps the AWS module in code that retries it if it fails.
4
+ #
5
+ module RetryAWS
6
+ class SimpleDB
7
+ def initialize(access_key, secret_key)
8
+ @simple_db = AWS::SimpleDB.new(access_key, secret_key)
9
+ @@log = STDOUT
10
+ end
11
+
12
+ def logger=(logger)
13
+ @@log = logger
14
+ end
15
+
16
+ private
17
+
18
+ def report_error(res)
19
+ @@log.puts "error #{$!}"
20
+ $@.each {|line| @@log.puts " #{line}"}
21
+ res
22
+ end
23
+
24
+ def retry?(err, retry_time)
25
+ if err.response.code >= 500 && retry_time < @retry_limit
26
+ sleep retry_time
27
+ return retry_time * 2
28
+ end
29
+ nil
30
+ end
31
+
32
+ public
33
+
34
+ def list_domains(max_domains = 100)
35
+ retry_time = 1
36
+ begin
37
+ @simple_db.list_domains(max_domains)
38
+ rescue AWS::ServiceError => err
39
+ retry if retry_time = retry?(err, retry_time)
40
+ report_error []
41
+ rescue
42
+ report_error []
43
+ end
44
+ end
45
+
46
+ def create_domain(domain_name)
47
+ retry_time = 1
48
+ begin
49
+ @simple_db.create_domain(domain_name)
50
+ rescue AWS::ServiceError => err
51
+ retry if retry_time = retry?(err, retry_time)
52
+ report_error false
53
+ rescue
54
+ report_error false
55
+ end
56
+ end
57
+
58
+ def delete_domain(domain_name)
59
+ retry_time = 1
60
+ begin
61
+ @simple_db.delete_domain(domain_name)
62
+ rescue AWS::ServiceError => err
63
+ retry if retry_time = retry?(err, retry_time)
64
+ report_error false
65
+ rescue
66
+ report_error false
67
+ end
68
+ end
69
+
70
+ def put_attributes(domain_name, item_name, attributes, replace=false)
71
+ retry_time = 1
72
+ begin
73
+ @simple_db.put_attributes(domain_name, item_name, attributes, replace)
74
+ rescue AWS::ServiceError => err
75
+ retry if retry_time = retry?(err, retry_time)
76
+ report_error false
77
+ rescue
78
+ report_error false
79
+ end
80
+ end
81
+
82
+ def delete_attributes(domain_name, item_name, attributes = {})
83
+ retry_time = 1
84
+ begin
85
+ @simple_db.delete_attributes(domain_name, item_name, attributes)
86
+ rescue AWS::ServiceError => err
87
+ retry if retry_time = retry?(err, retry_time)
88
+ report_error false
89
+ rescue
90
+ report_error false
91
+ end
92
+ end
93
+
94
+ def get_attributes(domain_name, item_name, attribute_name = nil)
95
+ retry_time = 1
96
+ begin
97
+ @simple_db.get_attributes(domain_name, item_name, attribute_name)
98
+ rescue AWS::ServiceError => err
99
+ retry if retry_time = retry?(err, retry_time)
100
+ report_error {}
101
+ rescue
102
+ report_error {}
103
+ end
104
+ end
105
+
106
+ def query(domain_name, query_expression=nil, options={:fetch_all=>true})
107
+ retry_time = 1
108
+ begin
109
+ @simple_db.query(domain_name, query_expression, options)
110
+ rescue AWS::ServiceError => err
111
+ retry if retry_time = retry?(err, retry_time)
112
+ report_error {}
113
+ rescue
114
+ report_error {}
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,63 @@
1
+ require 'AWS/EC2'
2
+
3
+ # SafeAWS wraps the AWS module in exception catcher blocks, so that any
4
+ # exceptions that are thrown do not affect the caller.
5
+ #
6
+ # The SafeEC2, SafeSQS, SafeSimpleDB and SafeS3 log any errors that they encounter, so
7
+ # that they can be examined later.
8
+ module SafeAWS
9
+ # Wrap EC2 functions that we use.
10
+ # Catch errors and do something reasonable.
11
+ class EC2
12
+ def initialize(access_key, secret_key)
13
+ @ec2 = AWS::EC2.new(access_key, secret_key)
14
+ @@log = STDOUT
15
+ end
16
+
17
+ def logger=(logger)
18
+ @@log = logger
19
+ end
20
+
21
+ private
22
+
23
+ def report_error(res)
24
+ @@log.puts "error #{$!}"
25
+ $@.each {|line| @@log.puts " #{line}"}
26
+ res
27
+ end
28
+
29
+ public
30
+
31
+ def describe_images(options={})
32
+ begin
33
+ @ec2.describe_images(options)
34
+ rescue
35
+ report_error []
36
+ end
37
+ end
38
+
39
+ def describe_instances(instance_ids=[])
40
+ begin
41
+ @ec2.describe_instances(instance_ids)
42
+ rescue
43
+ report_error []
44
+ end
45
+ end
46
+
47
+ def run_instances(image_id, min_count=1, max_count=min_count, options={})
48
+ begin
49
+ @ec2.run_instances(image_id, min_count, max_count, options)
50
+ rescue
51
+ report_error []
52
+ end
53
+ end
54
+
55
+ def terminate_instances(instance_ids = [])
56
+ begin
57
+ @ec2.terminate_instances(instance_ids)
58
+ rescue
59
+ report_error []
60
+ end
61
+ end
62
+ end
63
+ end
data/lib/SafeAWS/S3.rb ADDED
@@ -0,0 +1,56 @@
1
+ require 'AWS/S3'
2
+
3
+ module SafeAWS
4
+ class S3
5
+ def initialize(access_key, secret_key)
6
+ @s3 = AWS::S3.new(access_key, secret_key)
7
+ @@log = STDOUT
8
+ end
9
+
10
+ def logger=(logger)
11
+ @@log = logger
12
+ end
13
+
14
+ private
15
+
16
+ def report_error(res)
17
+ @@log.puts "error #{$!}"
18
+ $@.each {|line| @@log.puts " #{line}"}
19
+ res
20
+ end
21
+
22
+ public
23
+
24
+ # Create an S3 object.
25
+ # Catch errors, but if one occurs, return false.
26
+ def create_object(bucket_name, object_key, opts={})
27
+ begin
28
+ @s3.create_object(bucket_name, object_key, opts)
29
+ rescue
30
+ report_error false
31
+ end
32
+ end
33
+
34
+ def get_object(bucket_name, object_key, headers={})
35
+ begin
36
+ if block_given?
37
+ @s3.get_object(bucket_name, object_key, headers) do |segment|
38
+ yield(segment)
39
+ end
40
+ else
41
+ @s3.get_object(bucket_name, object_key, headers)
42
+ end
43
+ rescue
44
+ report_error nil
45
+ end
46
+ end
47
+
48
+ def set_canned_acl(canned_acl, bucket_name, object_key='')
49
+ begin
50
+ @s3.set_canned_acl(canned_acl, bucket_name, object_key)
51
+ rescue
52
+ report_error false
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,75 @@
1
+ require 'AWS/SQS'
2
+
3
+ module SafeAWS
4
+ # Wrap SQS routines that we use.
5
+ # Catch exceptions and return something reasonable.
6
+ class SQS
7
+ def initialize(*params)
8
+ @sqs = AWS::SQS.new(*params)
9
+ @@log = STDOUT
10
+ end
11
+
12
+ def logger=(logger)
13
+ @@log = logger
14
+ end
15
+
16
+ private
17
+
18
+ # report error and return result
19
+ def report_error(res)
20
+ @@log.puts "error #{$!}"
21
+ $@.each {|line| @@log.puts " #{line}"}
22
+ res
23
+ end
24
+
25
+ public
26
+
27
+ def create_queue(queue_name, visibility_timeout_secs = nil)
28
+ begin
29
+ @sqs.create_queue(queue_name, visibility_tmeout_secs)
30
+ rescue
31
+ report_error nil
32
+ end
33
+ end
34
+
35
+ def list_queues(queue_name_prefix = nil)
36
+ begin
37
+ @sqs.list_queues(queue_name_prefix)
38
+ rescue
39
+ report_error []
40
+ end
41
+ end
42
+
43
+ def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
44
+ begin
45
+ @sqs.receive_messages(queue_url, maximum, visibility_timeout)
46
+ rescue
47
+ report_error []
48
+ end
49
+ end
50
+
51
+ def send_message(queue_url, message_body, encode=false)
52
+ begin
53
+ @sqs.send_message(queue_url, message_body, encode)
54
+ rescue
55
+ report_error ''
56
+ end
57
+ end
58
+
59
+ def delete_message(queue_url, receipt_handle)
60
+ begin
61
+ @sqs.delete_message(queue_url, receipt_handle)
62
+ rescue
63
+ report_error false
64
+ end
65
+ end
66
+
67
+ def get_queue_attributes(queue_url, attribute='All')
68
+ begin
69
+ @sqs.get_queue_attributes(queue_url, attribute)
70
+ rescue
71
+ report_error {}
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,88 @@
1
+ require 'AWS/SimpleDB'
2
+
3
+ # SafeAWS wraps the AWS module in exception catcher blocks, so that any
4
+ # exceptions that are thrown do not affect the caller.
5
+ #
6
+ # The SafeEC2, SafeSQS, SafeSimpleDB, and SafeS3 log any errors that they encounter, so
7
+ # that they can be examined later.
8
+ module SafeAWS
9
+ # Wrap SimpleDB functions that we use.
10
+ # Catch errors and do something reasonable.
11
+ class SimpleDB
12
+ def initialize(access_key, secret_key)
13
+ @simple_db = AWS::SimpleDB.new(access_key, secret_key)
14
+ @@log = STDOUT
15
+ end
16
+
17
+ def logger=(logger)
18
+ @@log = logger
19
+ end
20
+
21
+ private
22
+
23
+ def report_error(res)
24
+ @@log.puts "error #{$!}"
25
+ $@.each {|line| @@log.puts " #{line}"}
26
+ res
27
+ end
28
+
29
+ public
30
+
31
+ def list_domains(max_domains = 100)
32
+ begin
33
+ @simple_db.list_domains(max_domains)
34
+ rescue
35
+ report_error []
36
+ end
37
+ end
38
+
39
+ def create_domain(domain_name)
40
+ begin
41
+ @simple_db.create_domain(domain_name)
42
+ rescue
43
+ report_error false
44
+ end
45
+ end
46
+
47
+ def delete_domain(domain_name)
48
+ begin
49
+ @simple_db.delete_domain(domain_name)
50
+ rescue
51
+ report_error false
52
+ end
53
+ end
54
+
55
+ def put_attributes(domain_name, item_name, attributes, replace=false)
56
+ begin
57
+ @simple_db.put_attributes(domain_name, item_name, attributes, replace)
58
+ rescue
59
+ report_error false
60
+ end
61
+ end
62
+
63
+ def delete_attributes(domain_name, item_name, attributes = {})
64
+ begin
65
+ @simple_db.delete_attributes(domain_name, item_name, attributes)
66
+ rescue
67
+ report_error false
68
+ end
69
+ end
70
+
71
+ def get_attributes(domain_name, item_name, attribute_name = nil)
72
+ begin
73
+ @simple_db.get_attributes(domain_name, item_name, attribute_name)
74
+ rescue
75
+ report_error {}
76
+ end
77
+ end
78
+
79
+ def query(domain_name, query_expression=nil, options={:fetch_all=>true})
80
+ begin
81
+ @simple_db.query(domain_name, query_expression, options)
82
+ rescue
83
+ report_error {}
84
+ end
85
+ end
86
+
87
+ end
88
+ end