staugaard-cloudmaster 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION.yml +4 -0
- data/bin/cloudmaster +45 -0
- data/lib/AWS/AWS.rb +3 -0
- data/lib/AWS/EC2.rb +14 -0
- data/lib/AWS/S3.rb +14 -0
- data/lib/AWS/SQS.rb +14 -0
- data/lib/AWS/SimpleDB.rb +14 -0
- data/lib/MockAWS/EC2.rb +119 -0
- data/lib/MockAWS/S3.rb +39 -0
- data/lib/MockAWS/SQS.rb +82 -0
- data/lib/MockAWS/SimpleDB.rb +46 -0
- data/lib/MockAWS/clock.rb +67 -0
- data/lib/OriginalAWS/AWS.rb +475 -0
- data/lib/OriginalAWS/EC2.rb +783 -0
- data/lib/OriginalAWS/S3.rb +559 -0
- data/lib/OriginalAWS/SQS.rb +159 -0
- data/lib/OriginalAWS/SimpleDB.rb +460 -0
- data/lib/RetryAWS/EC2.rb +88 -0
- data/lib/RetryAWS/S3.rb +77 -0
- data/lib/RetryAWS/SQS.rb +109 -0
- data/lib/RetryAWS/SimpleDB.rb +118 -0
- data/lib/SafeAWS/EC2.rb +63 -0
- data/lib/SafeAWS/S3.rb +56 -0
- data/lib/SafeAWS/SQS.rb +75 -0
- data/lib/SafeAWS/SimpleDB.rb +88 -0
- data/lib/aws_context.rb +165 -0
- data/lib/basic_configuration.rb +120 -0
- data/lib/clock.rb +10 -0
- data/lib/factory.rb +14 -0
- data/lib/file_logger.rb +36 -0
- data/lib/inifile.rb +148 -0
- data/lib/instance_logger.rb +25 -0
- data/lib/logger_factory.rb +38 -0
- data/lib/periodic.rb +29 -0
- data/lib/string_logger.rb +29 -0
- data/lib/sys_logger.rb +40 -0
- data/lib/user_data.rb +30 -0
- data/test/aws-config.ini +9 -0
- data/test/cloudmaster-tests.rb +329 -0
- data/test/configuration-test.rb +62 -0
- data/test/daytime-policy-tests.rb +47 -0
- data/test/enumerator-test.rb +47 -0
- data/test/fixed-policy-tests.rb +50 -0
- data/test/instance-pool-test.rb +359 -0
- data/test/instance-test.rb +98 -0
- data/test/job-policy-test.rb +95 -0
- data/test/manual-policy-tests.rb +63 -0
- data/test/named-queue-test.rb +90 -0
- data/test/resource-policy-tests.rb +126 -0
- data/test/suite +17 -0
- data/test/test-config.ini +47 -0
- metadata +111 -0
data/VERSION.yml
ADDED
data/bin/cloudmaster
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
$:.unshift File.join(ENV['AWS_HOME'], "app")
|
3
|
+
|
4
|
+
require 'configuration'
|
5
|
+
require 'aws_context'
|
6
|
+
require 'configuration'
|
7
|
+
require 'pool_runner'
|
8
|
+
require 'pp'
|
9
|
+
|
10
|
+
# Get optional args specifying what server to manage.
|
11
|
+
verbose = false
|
12
|
+
config_files = []
|
13
|
+
opts = []
|
14
|
+
args = ARGV
|
15
|
+
while args.size > 0
|
16
|
+
a = args.shift
|
17
|
+
case a
|
18
|
+
when "-?": puts "usage: cloudmaster [-v] [-l logfile] [-c configfile] [-p pool]"
|
19
|
+
exit 1
|
20
|
+
when "-v": verbose = true
|
21
|
+
when "-l": logfile = args.shift
|
22
|
+
when "-c": config_files << args.shift
|
23
|
+
when "-p": opts << args.shift.to_sym
|
24
|
+
end
|
25
|
+
end
|
26
|
+
opts.uniq!
|
27
|
+
|
28
|
+
LoggerFactory.setup(logfile, 'cloudmaster')
|
29
|
+
AwsContext.setup(:retry)
|
30
|
+
|
31
|
+
config = Cloudmaster::Configuration.new(config_files, opts)
|
32
|
+
|
33
|
+
if verbose
|
34
|
+
puts "Environment: #{config.aws[:aws_env]}" if config.aws[:aws_env]
|
35
|
+
puts "Log file: #{logfile}" if logfile
|
36
|
+
puts "Config file: #{config_files.join(' ')}" if config_files.size > 0
|
37
|
+
end
|
38
|
+
|
39
|
+
Signal.trap("TERM") do
|
40
|
+
# re-read the config file on SIGTERM -- into existing config object
|
41
|
+
config.refresh
|
42
|
+
end
|
43
|
+
|
44
|
+
# Run a pool supervisor
|
45
|
+
Cloudmaster::PoolRunner.new(config).run
|
data/lib/AWS/AWS.rb
ADDED
data/lib/AWS/EC2.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/EC2'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class EC2
|
6
|
+
def initialize(*args)
|
7
|
+
@ec2 = ::EC2.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@ec2.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/S3.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/S3'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class S3
|
6
|
+
def initialize(*args)
|
7
|
+
@s3 = ::S3.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args, &block)
|
11
|
+
@s3.send(key, *args, &block)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/SQS.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/SQS'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class SQS
|
6
|
+
def initialize(*args)
|
7
|
+
@sqs = ::SQS.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@sqs.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/AWS/SimpleDB.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
$:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
|
2
|
+
require 'OriginalAWS/SimpleDB'
|
3
|
+
|
4
|
+
module AWS
|
5
|
+
class SimpleDB
|
6
|
+
def initialize(*args)
|
7
|
+
@ec2 = ::SimpleDB.new(*args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(key, *args)
|
11
|
+
@ec2.send(key, *args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/MockAWS/EC2.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
|
2
|
+
# Mock SQS
|
3
|
+
# Provide just enough for cloudmaster.
|
4
|
+
module MockAWS
|
5
|
+
# Mock EC2
|
6
|
+
class EC2
|
7
|
+
@@log = STDOUT
|
8
|
+
@@images = [
|
9
|
+
{:owner_id=>"452272755447",
|
10
|
+
:state=>"available",
|
11
|
+
:id=>"ami-08856161",
|
12
|
+
:is_public=>false,
|
13
|
+
:location=>"chayden/chayden-ami-primes-test.img.manifest.xml"},
|
14
|
+
{:owner_id=>"452272755447",
|
15
|
+
:state=>"available",
|
16
|
+
:id=>"ami-3f856156",
|
17
|
+
:is_public=>false,
|
18
|
+
:location=>"chayden/chayden-ami-base.img.manifest.xml"},
|
19
|
+
{:owner_id=>"452272755447",
|
20
|
+
:state=>"available",
|
21
|
+
:id=>"ami-d18064b8",
|
22
|
+
:is_public=>false,
|
23
|
+
:location=>"chayden/chayden-ami-fibonacci-test.img.manifest.xml"},
|
24
|
+
]
|
25
|
+
|
26
|
+
@@running = []
|
27
|
+
@@instance_id = 0
|
28
|
+
|
29
|
+
def initialize(*params)
|
30
|
+
end
|
31
|
+
|
32
|
+
def describe_images(options={})
|
33
|
+
@@images
|
34
|
+
end
|
35
|
+
|
36
|
+
def describe_instances(instance_ids=[])
|
37
|
+
instances = []
|
38
|
+
if instance_ids.size > 0
|
39
|
+
instance_ids.each do |id|
|
40
|
+
instances += @@running.find_all {|r| r[:id] == id}
|
41
|
+
end
|
42
|
+
else
|
43
|
+
instances = @@running
|
44
|
+
end
|
45
|
+
[{:instances => instances}]
|
46
|
+
end
|
47
|
+
|
48
|
+
def run_instances(image_id, min_count=1, max_count=min_count, options={})
|
49
|
+
ids = []
|
50
|
+
max_count.times do
|
51
|
+
@@instance_id += 1
|
52
|
+
iid = "i-#{@@instance_id}"
|
53
|
+
starting = {:image_id => image_id, :id => iid,
|
54
|
+
:public_dns => "", :state => "running"}
|
55
|
+
@@log.puts "***** starting #{iid} #{image_id}"
|
56
|
+
@@running << starting
|
57
|
+
ids << {:id => iid}
|
58
|
+
end
|
59
|
+
{:instances => ids}
|
60
|
+
end
|
61
|
+
|
62
|
+
def terminate_instances(instance_ids = [])
|
63
|
+
instance_ids.each do |id|
|
64
|
+
@@log.puts "***** terminating #{id}"
|
65
|
+
@@running = @@running.find_all { |r| r[:id] != id }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
#######################################3333
|
70
|
+
# for testing
|
71
|
+
def logger=(logger)
|
72
|
+
@@log = logger
|
73
|
+
end
|
74
|
+
|
75
|
+
def pp_running
|
76
|
+
pp @@running
|
77
|
+
end
|
78
|
+
|
79
|
+
def count
|
80
|
+
@@running.size
|
81
|
+
end
|
82
|
+
|
83
|
+
def valid_ami_id
|
84
|
+
@@images[0][:id]
|
85
|
+
end
|
86
|
+
|
87
|
+
def set_public_dns(id, dns)
|
88
|
+
@@running.each do |inst|
|
89
|
+
if inst[:id] == id
|
90
|
+
inst[:public_dns] = dns
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def get_public_dns(id)
|
96
|
+
inst = @@running.find {|inst| inst[:id] == id}
|
97
|
+
inst.nil? ? "" : inst[:public_dns]
|
98
|
+
end
|
99
|
+
|
100
|
+
def set_state(id, state)
|
101
|
+
@@running.each do |inst|
|
102
|
+
if inst[:id] == id
|
103
|
+
inst[:state] = state
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def first_id
|
109
|
+
@@running.first[:id]
|
110
|
+
end
|
111
|
+
|
112
|
+
def reset
|
113
|
+
@@log = STDOUT
|
114
|
+
@@running = []
|
115
|
+
@@instance_id = 0
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
data/lib/MockAWS/S3.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
module MockAWS
|
2
|
+
# Mock S3
|
3
|
+
class S3
|
4
|
+
@@contents = {}
|
5
|
+
|
6
|
+
def initialize(*params)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Create an S3 object.
|
10
|
+
def create_object(bucket_name, object_key, opts={})
|
11
|
+
key = "#{bucket_name}/#{object_key}"
|
12
|
+
value = opts[:data]
|
13
|
+
@@contents[key] = value
|
14
|
+
end
|
15
|
+
|
16
|
+
# Return an object stored previously
|
17
|
+
def get_object(bucket_name, object_key, headers={})
|
18
|
+
key = "#{bucket_name}/#{object_key}"
|
19
|
+
@@contents[key]
|
20
|
+
end
|
21
|
+
|
22
|
+
def set_canned_acl(canned_acl, bucket_name, object_key='')
|
23
|
+
true
|
24
|
+
end
|
25
|
+
##############
|
26
|
+
# testing
|
27
|
+
def logger=(logger)
|
28
|
+
@@log = logger
|
29
|
+
end
|
30
|
+
|
31
|
+
def reset
|
32
|
+
@context = []
|
33
|
+
end
|
34
|
+
|
35
|
+
def contents
|
36
|
+
pp @@contents
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/MockAWS/SQS.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
module MockAWS
|
2
|
+
# Mock SQS
|
3
|
+
class SQS
|
4
|
+
@@log = STDOUT
|
5
|
+
@@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
|
6
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
|
7
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
|
8
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
|
9
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
|
10
|
+
|
11
|
+
@@messages = {}
|
12
|
+
@@queues.each {|q| @@messages[q] = []}
|
13
|
+
@@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
|
14
|
+
@@msg_seq = 1
|
15
|
+
|
16
|
+
def initialize(*params)
|
17
|
+
end
|
18
|
+
|
19
|
+
def logger=(logger)
|
20
|
+
@@log = logger
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_queue(queue_name, visibility_timeout_secs = nil)
|
24
|
+
@@queues << queue_name
|
25
|
+
@@queues.last
|
26
|
+
end
|
27
|
+
|
28
|
+
def list_queues(queue_name_prefix = nil)
|
29
|
+
filter = Regexp.new(queue_name_prefix)
|
30
|
+
@@queues.find_all {|q| q =~ filter}
|
31
|
+
end
|
32
|
+
|
33
|
+
def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
|
34
|
+
if @@messages[queue_url] == []
|
35
|
+
[]
|
36
|
+
else
|
37
|
+
visible = @@messages[queue_url].find_all {|m| m[:visible]}
|
38
|
+
@@log.puts "***** receiving #{visible.first[:id]} #{visible.first[:body]}"
|
39
|
+
[visible.first]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def send_message(queue_url, message_body, encode=false)
|
44
|
+
mid = "id-#{@@msg_seq}"
|
45
|
+
message = { :id => mid,
|
46
|
+
:body => message_body,
|
47
|
+
:visible => true}
|
48
|
+
@@msg_seq += 1
|
49
|
+
@@messages[queue_url] = [] unless @@messages[queue_url]
|
50
|
+
@@messages[queue_url] << message
|
51
|
+
@@log.puts "***** sending #{File.basename(queue_url)} #{mid} => #{message_body}"
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
def delete_message(queue_url, receipt_handle)
|
56
|
+
@@messages[queue_url] = @@messages[queue_url].reject {|m| m[:id] == receipt_handle}
|
57
|
+
true
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
def get_queue_attributes(queue_url, attribute='All')
|
62
|
+
@@attributes["ApproximateNumberOfMessages"] = @@messages[queue_url].size
|
63
|
+
@@attributes
|
64
|
+
end
|
65
|
+
|
66
|
+
######################
|
67
|
+
# testing
|
68
|
+
def reset
|
69
|
+
@@log = STDOUT
|
70
|
+
@@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
|
71
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
|
72
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
|
73
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
|
74
|
+
"http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
|
75
|
+
|
76
|
+
@@messages = {}
|
77
|
+
@@queues.each {|q| @@messages[q] = []}
|
78
|
+
@@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
|
79
|
+
@@msg_seq = 1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'AWS/SimpleDB'
|
2
|
+
|
3
|
+
module MockAWS
|
4
|
+
class SimpleDB
|
5
|
+
def initialize(*params)
|
6
|
+
end
|
7
|
+
|
8
|
+
def list_domains(max_domains = 100)
|
9
|
+
[]
|
10
|
+
end
|
11
|
+
|
12
|
+
def create_domain(domain_name)
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete_domain(domain_name)
|
17
|
+
true
|
18
|
+
end
|
19
|
+
|
20
|
+
def put_attributes(domain_name, item_name, attributes, replace=false)
|
21
|
+
true
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete_attributes(domain_name, item_name, attributes = {})
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
def get_attributes(domain_name, item_name, attribute_name = nil)
|
29
|
+
true
|
30
|
+
end
|
31
|
+
|
32
|
+
def query(domain_name, query_expression=nil, options={:fetch_all=>true})
|
33
|
+
true
|
34
|
+
end
|
35
|
+
##############
|
36
|
+
# testing
|
37
|
+
def logger=(logger)
|
38
|
+
@@log = logger
|
39
|
+
end
|
40
|
+
|
41
|
+
def reset
|
42
|
+
@context = []
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
|
2
|
+
# Used to mock Time
|
3
|
+
|
4
|
+
class Clock
|
5
|
+
attr_accessor :time
|
6
|
+
include Comparable
|
7
|
+
|
8
|
+
@@current_time = 0
|
9
|
+
|
10
|
+
def initialize(time)
|
11
|
+
@time = time
|
12
|
+
end
|
13
|
+
|
14
|
+
def Clock.at(time)
|
15
|
+
Clock.new(time)
|
16
|
+
end
|
17
|
+
|
18
|
+
def Clock.now
|
19
|
+
Clock.new(@@current_time)
|
20
|
+
end
|
21
|
+
|
22
|
+
def Clock.hour
|
23
|
+
(@@current_time/3600).floor
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_s
|
27
|
+
@time.to_s
|
28
|
+
end
|
29
|
+
|
30
|
+
def +(time)
|
31
|
+
Clock.new(time + @time)
|
32
|
+
end
|
33
|
+
|
34
|
+
def -(clock)
|
35
|
+
@time - clock.time
|
36
|
+
end
|
37
|
+
|
38
|
+
def <=>(other)
|
39
|
+
@time <=> other.time
|
40
|
+
end
|
41
|
+
|
42
|
+
def strftime(format)
|
43
|
+
@time.to_s
|
44
|
+
end
|
45
|
+
|
46
|
+
def Clock.sleep(numeric = 0)
|
47
|
+
@@current_time += numeric
|
48
|
+
end
|
49
|
+
|
50
|
+
def Clock.xmlschema(tm)
|
51
|
+
Clock.new(Time.xmlschema(tm).to_i)
|
52
|
+
end
|
53
|
+
|
54
|
+
def Clock.parse(tm)
|
55
|
+
Clock.new(Time.parse(tm).to_i)
|
56
|
+
end
|
57
|
+
|
58
|
+
##############
|
59
|
+
# testing
|
60
|
+
def Clock.reset
|
61
|
+
@@current_time = 0
|
62
|
+
end
|
63
|
+
|
64
|
+
def Clock.set(numeric)
|
65
|
+
@@current_time = numeric
|
66
|
+
end
|
67
|
+
end
|