staugaard-cloudmaster 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION.yml +4 -0
- data/bin/cloudmaster +45 -0
- data/lib/AWS/AWS.rb +3 -0
- data/lib/AWS/EC2.rb +14 -0
- data/lib/AWS/S3.rb +14 -0
- data/lib/AWS/SQS.rb +14 -0
- data/lib/AWS/SimpleDB.rb +14 -0
- data/lib/MockAWS/EC2.rb +119 -0
- data/lib/MockAWS/S3.rb +39 -0
- data/lib/MockAWS/SQS.rb +82 -0
- data/lib/MockAWS/SimpleDB.rb +46 -0
- data/lib/MockAWS/clock.rb +67 -0
- data/lib/OriginalAWS/AWS.rb +475 -0
- data/lib/OriginalAWS/EC2.rb +783 -0
- data/lib/OriginalAWS/S3.rb +559 -0
- data/lib/OriginalAWS/SQS.rb +159 -0
- data/lib/OriginalAWS/SimpleDB.rb +460 -0
- data/lib/RetryAWS/EC2.rb +88 -0
- data/lib/RetryAWS/S3.rb +77 -0
- data/lib/RetryAWS/SQS.rb +109 -0
- data/lib/RetryAWS/SimpleDB.rb +118 -0
- data/lib/SafeAWS/EC2.rb +63 -0
- data/lib/SafeAWS/S3.rb +56 -0
- data/lib/SafeAWS/SQS.rb +75 -0
- data/lib/SafeAWS/SimpleDB.rb +88 -0
- data/lib/aws_context.rb +165 -0
- data/lib/basic_configuration.rb +120 -0
- data/lib/clock.rb +10 -0
- data/lib/factory.rb +14 -0
- data/lib/file_logger.rb +36 -0
- data/lib/inifile.rb +148 -0
- data/lib/instance_logger.rb +25 -0
- data/lib/logger_factory.rb +38 -0
- data/lib/periodic.rb +29 -0
- data/lib/string_logger.rb +29 -0
- data/lib/sys_logger.rb +40 -0
- data/lib/user_data.rb +30 -0
- data/test/aws-config.ini +9 -0
- data/test/cloudmaster-tests.rb +329 -0
- data/test/configuration-test.rb +62 -0
- data/test/daytime-policy-tests.rb +47 -0
- data/test/enumerator-test.rb +47 -0
- data/test/fixed-policy-tests.rb +50 -0
- data/test/instance-pool-test.rb +359 -0
- data/test/instance-test.rb +98 -0
- data/test/job-policy-test.rb +95 -0
- data/test/manual-policy-tests.rb +63 -0
- data/test/named-queue-test.rb +90 -0
- data/test/resource-policy-tests.rb +126 -0
- data/test/suite +17 -0
- data/test/test-config.ini +47 -0
- metadata +111 -0
data/VERSION.yml
ADDED
data/bin/cloudmaster
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
$:.unshift File.join(ENV['AWS_HOME'], "app")
|
3
|
+
|
4
|
+
require 'configuration'
|
5
|
+
require 'aws_context'
|
6
|
+
require 'configuration'
|
7
|
+
require 'pool_runner'
|
8
|
+
require 'pp'
|
9
|
+
|
10
|
+
# Get optional args specifying what server to manage.
|
11
|
+
verbose = false
|
12
|
+
config_files = []
|
13
|
+
opts = []
|
14
|
+
args = ARGV
|
15
|
+
while args.size > 0
|
16
|
+
a = args.shift
|
17
|
+
case a
|
18
|
+
when "-?": puts "usage: cloudmaster [-v] [-l logfile] [-c configfile] [-p pool]"
|
19
|
+
exit 1
|
20
|
+
when "-v": verbose = true
|
21
|
+
when "-l": logfile = args.shift
|
22
|
+
when "-c": config_files << args.shift
|
23
|
+
when "-p": opts << args.shift.to_sym
|
24
|
+
end
|
25
|
+
end
|
26
|
+
opts.uniq!
|
27
|
+
|
28
|
+
LoggerFactory.setup(logfile, 'cloudmaster')
|
29
|
+
AwsContext.setup(:retry)
|
30
|
+
|
31
|
+
config = Cloudmaster::Configuration.new(config_files, opts)
|
32
|
+
|
33
|
+
if verbose
|
34
|
+
puts "Environment: #{config.aws[:aws_env]}" if config.aws[:aws_env]
|
35
|
+
puts "Log file: #{logfile}" if logfile
|
36
|
+
puts "Config file: #{config_files.join(' ')}" if config_files.size > 0
|
37
|
+
end
|
38
|
+
|
39
|
+
Signal.trap("TERM") do
|
40
|
+
# re-read the config file on SIGTERM -- into existing config object
|
41
|
+
config.refresh
|
42
|
+
end
|
43
|
+
|
44
|
+
# Run a pool supervisor
|
45
|
+
Cloudmaster::PoolRunner.new(config).run
|
data/lib/AWS/AWS.rb
ADDED
data/lib/AWS/EC2.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/EC2'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class EC2
|
6
|
+
def initialize(*args)
|
7
|
+
@ec2 = ::EC2.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@ec2.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/S3.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/S3'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class S3
|
6
|
+
def initialize(*args)
|
7
|
+
@s3 = ::S3.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args, &block)
|
11
|
+
@s3.send(key, *args, &block)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/SQS.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/SQS'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class SQS
|
6
|
+
def initialize(*args)
|
7
|
+
@sqs = ::SQS.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@sqs.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/SimpleDB.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/SimpleDB'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class SimpleDB
|
6
|
+
def initialize(*args)
|
7
|
+
@ec2 = ::SimpleDB.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@ec2.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/MockAWS/EC2.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
|
2
|
+
# Mock SQS
|
3
|
+
# Provide just enough for cloudmaster.
|
4
|
+
module MockAWS
|
5
|
+
# Mock EC2
|
6
|
+
class EC2
|
7
|
+
@@log = STDOUT
|
8
|
+
@@images = [
|
9
|
+
{:owner_id=>"452272755447",
|
10
|
+
:state=>"available",
|
11
|
+
:id=>"ami-08856161",
|
12
|
+
:is_public=>false,
|
13
|
+
:location=>"chayden/chayden-ami-primes-test.img.manifest.xml"},
|
14
|
+
{:owner_id=>"452272755447",
|
15
|
+
:state=>"available",
|
16
|
+
:id=>"ami-3f856156",
|
17
|
+
:is_public=>false,
|
18
|
+
:location=>"chayden/chayden-ami-base.img.manifest.xml"},
|
19
|
+
{:owner_id=>"452272755447",
|
20
|
+
:state=>"available",
|
21
|
+
:id=>"ami-d18064b8",
|
22
|
+
:is_public=>false,
|
23
|
+
:location=>"chayden/chayden-ami-fibonacci-test.img.manifest.xml"},
|
24
|
+
]
|
25
|
+
|
26
|
+
@@running = []
|
27
|
+
@@instance_id = 0
|
28
|
+
|
29
|
+
def initialize(*params)
|
30
|
+
end
|
31
|
+
|
32
|
+
def describe_images(options={})
|
33
|
+
@@images
|
34
|
+
end
|
35
|
+
|
36
|
+
def describe_instances(instance_ids=[])
|
37
|
+
instances = []
|
38
|
+
if instance_ids.size > 0
|
39
|
+
instance_ids.each do |id|
|
40
|
+
instances += @@running.find_all {|r| r[:id] == id}
|
41
|
+
end
|
42
|
+
else
|
43
|
+
instances = @@running
|
44
|
+
end
|
45
|
+
[{:instances => instances}]
|
46
|
+
end
|
47
|
+
|
48
|
+
def run_instances(image_id, min_count=1, max_count=min_count, options={})
|
49
|
+
ids = []
|
50
|
+
max_count.times do
|
51
|
+
@@instance_id += 1
|
52
|
+
iid = "i-#{@@instance_id}"
|
53
|
+
starting = {:image_id => image_id, :id => iid,
|
54
|
+
:public_dns => "", :state => "running"}
|
55
|
+
@@log.puts "***** starting #{iid} #{image_id}"
|
56
|
+
@@running << starting
|
57
|
+
ids << {:id => iid}
|
58
|
+
end
|
59
|
+
{:instances => ids}
|
60
|
+
end
|
61
|
+
|
62
|
+
def terminate_instances(instance_ids = [])
|
63
|
+
instance_ids.each do |id|
|
64
|
+
@@log.puts "***** terminating #{id}"
|
65
|
+
@@running = @@running.find_all { |r| r[:id] != id }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
#######################################3333
|
70
|
+
# for testing
|
71
|
+
def logger=(logger)
|
72
|
+
@@log = logger
|
73
|
+
end
|
74
|
+
|
75
|
+
def pp_running
|
76
|
+
pp @@running
|
77
|
+
end
|
78
|
+
|
79
|
+
def count
|
80
|
+
@@running.size
|
81
|
+
end
|
82
|
+
|
83
|
+
def valid_ami_id
|
84
|
+
@@images[0][:id]
|
85
|
+
end
|
86
|
+
|
87
|
+
def set_public_dns(id, dns)
|
88
|
+
@@running.each do |inst|
|
89
|
+
if inst[:id] == id
|
90
|
+
inst[:public_dns] = dns
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def get_public_dns(id)
|
96
|
+
inst = @@running.find {|inst| inst[:id] == id}
|
97
|
+
inst.nil? ? "" : inst[:public_dns]
|
98
|
+
end
|
99
|
+
|
100
|
+
def set_state(id, state)
|
101
|
+
@@running.each do |inst|
|
102
|
+
if inst[:id] == id
|
103
|
+
inst[:state] = state
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def first_id
|
109
|
+
@@running.first[:id]
|
110
|
+
end
|
111
|
+
|
112
|
+
def reset
|
113
|
+
@@log = STDOUT
|
114
|
+
@@running = []
|
115
|
+
@@instance_id = 0
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
data/lib/MockAWS/S3.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
module MockAWS
|
2
|
+
# Mock S3
|
3
|
+
class S3
|
4
|
+
@@contents = {}
|
5
|
+
|
6
|
+
def initialize(*params)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Create an S3 object.
|
10
|
+
def create_object(bucket_name, object_key, opts={})
|
11
|
+
key = "#{bucket_name}/#{object_key}"
|
12
|
+
value = opts[:data]
|
13
|
+
@@contents[key] = value
|
14
|
+
end
|
15
|
+
|
16
|
+
# Return an object stored previously
|
17
|
+
def get_object(bucket_name, object_key, headers={})
|
18
|
+
key = "#{bucket_name}/#{object_key}"
|
19
|
+
@@contents[key]
|
20
|
+
end
|
21
|
+
|
22
|
+
def set_canned_acl(canned_acl, bucket_name, object_key='')
|
23
|
+
true
|
24
|
+
end
|
25
|
+
##############
|
26
|
+
# testing
|
27
|
+
def logger=(logger)
|
28
|
+
@@log = logger
|
29
|
+
end
|
30
|
+
|
31
|
+
def reset
|
32
|
+
@context = []
|
33
|
+
end
|
34
|
+
|
35
|
+
def contents
|
36
|
+
pp @@contents
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/MockAWS/SQS.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
module MockAWS
|
2
|
+
# Mock SQS
|
3
|
+
class SQS
|
4
|
+
@@log = STDOUT
|
5
|
+
@@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
|
6
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
|
7
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
|
8
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
|
9
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
|
10
|
+
|
11
|
+
@@messages = {}
|
12
|
+
@@queues.each {|q| @@messages[q] = []}
|
13
|
+
@@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
|
14
|
+
@@msg_seq = 1
|
15
|
+
|
16
|
+
def initialize(*params)
|
17
|
+
end
|
18
|
+
|
19
|
+
def logger=(logger)
|
20
|
+
@@log = logger
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_queue(queue_name, visibility_timeout_secs = nil)
|
24
|
+
@@queues << queue_name
|
25
|
+
@@queues.last
|
26
|
+
end
|
27
|
+
|
28
|
+
def list_queues(queue_name_prefix = nil)
|
29
|
+
filter = Regexp.new(queue_name_prefix)
|
30
|
+
@@queues.find_all {|q| q =~ filter}
|
31
|
+
end
|
32
|
+
|
33
|
+
def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
|
34
|
+
if @@messages[queue_url] == []
|
35
|
+
[]
|
36
|
+
else
|
37
|
+
visible = @@messages[queue_url].find_all {|m| m[:visible]}
|
38
|
+
@@log.puts "***** receiving #{visible.first[:id]} #{visible.first[:body]}"
|
39
|
+
[visible.first]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def send_message(queue_url, message_body, encode=false)
|
44
|
+
mid = "id-#{@@msg_seq}"
|
45
|
+
message = { :id => mid,
|
46
|
+
:body => message_body,
|
47
|
+
:visible => true}
|
48
|
+
@@msg_seq += 1
|
49
|
+
@@messages[queue_url] = [] unless @@messages[queue_url]
|
50
|
+
@@messages[queue_url] << message
|
51
|
+
@@log.puts "***** sending #{File.basename(queue_url)} #{mid} => #{message_body}"
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
def delete_message(queue_url, receipt_handle)
|
56
|
+
@@messages[queue_url] = @@messages[queue_url].reject {|m| m[:id] == receipt_handle}
|
57
|
+
true
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
def get_queue_attributes(queue_url, attribute='All')
|
62
|
+
@@attributes["ApproximateNumberOfMessages"] = @@messages[queue_url].size
|
63
|
+
@@attributes
|
64
|
+
end
|
65
|
+
|
66
|
+
######################
|
67
|
+
# testing
|
68
|
+
def reset
|
69
|
+
@@log = STDOUT
|
70
|
+
@@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
|
71
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
|
72
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
|
73
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
|
74
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
|
75
|
+
|
76
|
+
@@messages = {}
|
77
|
+
@@queues.each {|q| @@messages[q] = []}
|
78
|
+
@@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
|
79
|
+
@@msg_seq = 1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'AWS/SimpleDB'
|
2
|
+
|
3
|
+
module MockAWS
|
4
|
+
class SimpleDB
|
5
|
+
def initialize(*params)
|
6
|
+
end
|
7
|
+
|
8
|
+
def list_domains(max_domains = 100)
|
9
|
+
[]
|
10
|
+
end
|
11
|
+
|
12
|
+
def create_domain(domain_name)
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete_domain(domain_name)
|
17
|
+
true
|
18
|
+
end
|
19
|
+
|
20
|
+
def put_attributes(domain_name, item_name, attributes, replace=false)
|
21
|
+
true
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete_attributes(domain_name, item_name, attributes = {})
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
def get_attributes(domain_name, item_name, attribute_name = nil)
|
29
|
+
true
|
30
|
+
end
|
31
|
+
|
32
|
+
def query(domain_name, query_expression=nil, options={:fetch_all=>true})
|
33
|
+
true
|
34
|
+
end
|
35
|
+
##############
|
36
|
+
# testing
|
37
|
+
def logger=(logger)
|
38
|
+
@@log = logger
|
39
|
+
end
|
40
|
+
|
41
|
+
def reset
|
42
|
+
@context = []
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
|
2
|
+
# Used to mock Time
|
3
|
+
|
4
|
+
class Clock
|
5
|
+
attr_accessor :time
|
6
|
+
include Comparable
|
7
|
+
|
8
|
+
@@current_time = 0
|
9
|
+
|
10
|
+
def initialize(time)
|
11
|
+
@time = time
|
12
|
+
end
|
13
|
+
|
14
|
+
def Clock.at(time)
|
15
|
+
Clock.new(time)
|
16
|
+
end
|
17
|
+
|
18
|
+
def Clock.now
|
19
|
+
Clock.new(@@current_time)
|
20
|
+
end
|
21
|
+
|
22
|
+
def Clock.hour
|
23
|
+
(@@current_time/3600).floor
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_s
|
27
|
+
@time.to_s
|
28
|
+
end
|
29
|
+
|
30
|
+
def +(time)
|
31
|
+
Clock.new(time + @time)
|
32
|
+
end
|
33
|
+
|
34
|
+
def -(clock)
|
35
|
+
@time - clock.time
|
36
|
+
end
|
37
|
+
|
38
|
+
def <=>(other)
|
39
|
+
@time <=> other.time
|
40
|
+
end
|
41
|
+
|
42
|
+
def strftime(format)
|
43
|
+
@time.to_s
|
44
|
+
end
|
45
|
+
|
46
|
+
def Clock.sleep(numeric = 0)
|
47
|
+
@@current_time += numeric
|
48
|
+
end
|
49
|
+
|
50
|
+
def Clock.xmlschema(tm)
|
51
|
+
Clock.new(Time.xmlschema(tm).to_i)
|
52
|
+
end
|
53
|
+
|
54
|
+
def Clock.parse(tm)
|
55
|
+
Clock.new(Time.parse(tm).to_i)
|
56
|
+
end
|
57
|
+
|
58
|
+
##############
|
59
|
+
# testing
|
60
|
+
def Clock.reset
|
61
|
+
@@current_time = 0
|
62
|
+
end
|
63
|
+
|
64
|
+
def Clock.set(numeric)
|
65
|
+
@@current_time = numeric
|
66
|
+
end
|
67
|
+
end
|