staugaard-cloudmaster 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/VERSION.yml +4 -0
  2. data/bin/cloudmaster +45 -0
  3. data/lib/AWS/AWS.rb +3 -0
  4. data/lib/AWS/EC2.rb +14 -0
  5. data/lib/AWS/S3.rb +14 -0
  6. data/lib/AWS/SQS.rb +14 -0
  7. data/lib/AWS/SimpleDB.rb +14 -0
  8. data/lib/MockAWS/EC2.rb +119 -0
  9. data/lib/MockAWS/S3.rb +39 -0
  10. data/lib/MockAWS/SQS.rb +82 -0
  11. data/lib/MockAWS/SimpleDB.rb +46 -0
  12. data/lib/MockAWS/clock.rb +67 -0
  13. data/lib/OriginalAWS/AWS.rb +475 -0
  14. data/lib/OriginalAWS/EC2.rb +783 -0
  15. data/lib/OriginalAWS/S3.rb +559 -0
  16. data/lib/OriginalAWS/SQS.rb +159 -0
  17. data/lib/OriginalAWS/SimpleDB.rb +460 -0
  18. data/lib/RetryAWS/EC2.rb +88 -0
  19. data/lib/RetryAWS/S3.rb +77 -0
  20. data/lib/RetryAWS/SQS.rb +109 -0
  21. data/lib/RetryAWS/SimpleDB.rb +118 -0
  22. data/lib/SafeAWS/EC2.rb +63 -0
  23. data/lib/SafeAWS/S3.rb +56 -0
  24. data/lib/SafeAWS/SQS.rb +75 -0
  25. data/lib/SafeAWS/SimpleDB.rb +88 -0
  26. data/lib/aws_context.rb +165 -0
  27. data/lib/basic_configuration.rb +120 -0
  28. data/lib/clock.rb +10 -0
  29. data/lib/factory.rb +14 -0
  30. data/lib/file_logger.rb +36 -0
  31. data/lib/inifile.rb +148 -0
  32. data/lib/instance_logger.rb +25 -0
  33. data/lib/logger_factory.rb +38 -0
  34. data/lib/periodic.rb +29 -0
  35. data/lib/string_logger.rb +29 -0
  36. data/lib/sys_logger.rb +40 -0
  37. data/lib/user_data.rb +30 -0
  38. data/test/aws-config.ini +9 -0
  39. data/test/cloudmaster-tests.rb +329 -0
  40. data/test/configuration-test.rb +62 -0
  41. data/test/daytime-policy-tests.rb +47 -0
  42. data/test/enumerator-test.rb +47 -0
  43. data/test/fixed-policy-tests.rb +50 -0
  44. data/test/instance-pool-test.rb +359 -0
  45. data/test/instance-test.rb +98 -0
  46. data/test/job-policy-test.rb +95 -0
  47. data/test/manual-policy-tests.rb +63 -0
  48. data/test/named-queue-test.rb +90 -0
  49. data/test/resource-policy-tests.rb +126 -0
  50. data/test/suite +17 -0
  51. data/test/test-config.ini +47 -0
  52. metadata +111 -0
data/VERSION.yml ADDED
@@ -0,0 +1,4 @@
1
+ ---
2
+ :major: 0
3
+ :minor: 1
4
+ :patch: 1
data/bin/cloudmaster ADDED
@@ -0,0 +1,45 @@
1
+ #!/usr/bin/ruby
2
+ $:.unshift File.join(ENV['AWS_HOME'], "app")
3
+
4
+ require 'configuration'
5
+ require 'aws_context'
6
+ require 'configuration'
7
+ require 'pool_runner'
8
+ require 'pp'
9
+
10
+ # Get optional args specifying what server to manage.
11
+ verbose = false
12
+ config_files = []
13
+ opts = []
14
+ args = ARGV
15
+ while args.size > 0
16
+ a = args.shift
17
+ case a
18
+ when "-?": puts "usage: cloudmaster [-v] [-l logfile] [-c configfile] [-p pool]"
19
+ exit 1
20
+ when "-v": verbose = true
21
+ when "-l": logfile = args.shift
22
+ when "-c": config_files << args.shift
23
+ when "-p": opts << args.shift.to_sym
24
+ end
25
+ end
26
+ opts.uniq!
27
+
28
+ LoggerFactory.setup(logfile, 'cloudmaster')
29
+ AwsContext.setup(:retry)
30
+
31
+ config = Cloudmaster::Configuration.new(config_files, opts)
32
+
33
+ if verbose
34
+ puts "Environment: #{config.aws[:aws_env]}" if config.aws[:aws_env]
35
+ puts "Log file: #{logfile}" if logfile
36
+ puts "Config file: #{config_files.join(' ')}" if config_files.size > 0
37
+ end
38
+
39
+ Signal.trap("TERM") do
40
+ # re-read the config file on SIGTERM -- into existing config object
41
+ config.refresh
42
+ end
43
+
44
+ # Run a pool supervisor
45
+ Cloudmaster::PoolRunner.new(config).run
data/lib/AWS/AWS.rb ADDED
@@ -0,0 +1,3 @@
1
+
2
+ require '../OriginalAWS/AWS'
3
+
data/lib/AWS/EC2.rb ADDED
@@ -0,0 +1,14 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
2
+ require 'OriginalAWS/EC2'
3
+
4
+ module AWS
5
+ class EC2
6
+ def initialize(*args)
7
+ @ec2 = ::EC2.new(*args)
8
+ end
9
+
10
+ def method_missing(key, *args)
11
+ @ec2.send(key, *args)
12
+ end
13
+ end
14
+ end
data/lib/AWS/S3.rb ADDED
@@ -0,0 +1,14 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
2
+ require 'OriginalAWS/S3'
3
+
4
+ module AWS
5
+ class S3
6
+ def initialize(*args)
7
+ @s3 = ::S3.new(*args)
8
+ end
9
+
10
+ def method_missing(key, *args, &block)
11
+ @s3.send(key, *args, &block)
12
+ end
13
+ end
14
+ end
data/lib/AWS/SQS.rb ADDED
@@ -0,0 +1,14 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
2
+ require 'OriginalAWS/SQS'
3
+
4
+ module AWS
5
+ class SQS
6
+ def initialize(*args)
7
+ @sqs = ::SQS.new(*args)
8
+ end
9
+
10
+ def method_missing(key, *args)
11
+ @sqs.send(key, *args)
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ $:.unshift(File.join(ENV['AWS_HOME'], "lib", "OriginalAWS"))
2
+ require 'OriginalAWS/SimpleDB'
3
+
4
+ module AWS
5
+ class SimpleDB
6
+ def initialize(*args)
7
+ @ec2 = ::SimpleDB.new(*args)
8
+ end
9
+
10
+ def method_missing(key, *args)
11
+ @ec2.send(key, *args)
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,119 @@
1
+
2
+ # Mock SQS
3
+ # Provide just enough for cloudmaster.
4
+ module MockAWS
5
+ # Mock EC2
6
+ class EC2
7
+ @@log = STDOUT
8
+ @@images = [
9
+ {:owner_id=>"452272755447",
10
+ :state=>"available",
11
+ :id=>"ami-08856161",
12
+ :is_public=>false,
13
+ :location=>"chayden/chayden-ami-primes-test.img.manifest.xml"},
14
+ {:owner_id=>"452272755447",
15
+ :state=>"available",
16
+ :id=>"ami-3f856156",
17
+ :is_public=>false,
18
+ :location=>"chayden/chayden-ami-base.img.manifest.xml"},
19
+ {:owner_id=>"452272755447",
20
+ :state=>"available",
21
+ :id=>"ami-d18064b8",
22
+ :is_public=>false,
23
+ :location=>"chayden/chayden-ami-fibonacci-test.img.manifest.xml"},
24
+ ]
25
+
26
+ @@running = []
27
+ @@instance_id = 0
28
+
29
+ def initialize(*params)
30
+ end
31
+
32
+ def describe_images(options={})
33
+ @@images
34
+ end
35
+
36
+ def describe_instances(instance_ids=[])
37
+ instances = []
38
+ if instance_ids.size > 0
39
+ instance_ids.each do |id|
40
+ instances += @@running.find_all {|r| r[:id] == id}
41
+ end
42
+ else
43
+ instances = @@running
44
+ end
45
+ [{:instances => instances}]
46
+ end
47
+
48
+ def run_instances(image_id, min_count=1, max_count=min_count, options={})
49
+ ids = []
50
+ max_count.times do
51
+ @@instance_id += 1
52
+ iid = "i-#{@@instance_id}"
53
+ starting = {:image_id => image_id, :id => iid,
54
+ :public_dns => "", :state => "running"}
55
+ @@log.puts "***** starting #{iid} #{image_id}"
56
+ @@running << starting
57
+ ids << {:id => iid}
58
+ end
59
+ {:instances => ids}
60
+ end
61
+
62
+ def terminate_instances(instance_ids = [])
63
+ instance_ids.each do |id|
64
+ @@log.puts "***** terminating #{id}"
65
+ @@running = @@running.find_all { |r| r[:id] != id }
66
+ end
67
+ end
68
+
69
+ #######################################3333
70
+ # for testing
71
+ def logger=(logger)
72
+ @@log = logger
73
+ end
74
+
75
+ def pp_running
76
+ pp @@running
77
+ end
78
+
79
+ def count
80
+ @@running.size
81
+ end
82
+
83
+ def valid_ami_id
84
+ @@images[0][:id]
85
+ end
86
+
87
+ def set_public_dns(id, dns)
88
+ @@running.each do |inst|
89
+ if inst[:id] == id
90
+ inst[:public_dns] = dns
91
+ end
92
+ end
93
+ end
94
+
95
+ def get_public_dns(id)
96
+ inst = @@running.find {|inst| inst[:id] == id}
97
+ inst.nil? ? "" : inst[:public_dns]
98
+ end
99
+
100
+ def set_state(id, state)
101
+ @@running.each do |inst|
102
+ if inst[:id] == id
103
+ inst[:state] = state
104
+ end
105
+ end
106
+ end
107
+
108
+ def first_id
109
+ @@running.first[:id]
110
+ end
111
+
112
+ def reset
113
+ @@log = STDOUT
114
+ @@running = []
115
+ @@instance_id = 0
116
+ end
117
+
118
+ end
119
+ end
data/lib/MockAWS/S3.rb ADDED
@@ -0,0 +1,39 @@
1
+ module MockAWS
2
+ # Mock S3
3
+ class S3
4
+ @@contents = {}
5
+
6
+ def initialize(*params)
7
+ end
8
+
9
+ # Create an S3 object.
10
+ def create_object(bucket_name, object_key, opts={})
11
+ key = "#{bucket_name}/#{object_key}"
12
+ value = opts[:data]
13
+ @@contents[key] = value
14
+ end
15
+
16
+ # Return an object stored previously
17
+ def get_object(bucket_name, object_key, headers={})
18
+ key = "#{bucket_name}/#{object_key}"
19
+ @@contents[key]
20
+ end
21
+
22
+ def set_canned_acl(canned_acl, bucket_name, object_key='')
23
+ true
24
+ end
25
+ ##############
26
+ # testing
27
+ def logger=(logger)
28
+ @@log = logger
29
+ end
30
+
31
+ def reset
32
+ @context = []
33
+ end
34
+
35
+ def contents
36
+ pp @@contents
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,82 @@
1
+ module MockAWS
2
+ # Mock SQS
3
+ class SQS
4
+ @@log = STDOUT
5
+ @@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
6
+ "http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
7
+ "http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
8
+ "http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
9
+ "http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
10
+
11
+ @@messages = {}
12
+ @@queues.each {|q| @@messages[q] = []}
13
+ @@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
14
+ @@msg_seq = 1
15
+
16
+ def initialize(*params)
17
+ end
18
+
19
+ def logger=(logger)
20
+ @@log = logger
21
+ end
22
+
23
+ def create_queue(queue_name, visibility_timeout_secs = nil)
24
+ @@queues << queue_name
25
+ @@queues.last
26
+ end
27
+
28
+ def list_queues(queue_name_prefix = nil)
29
+ filter = Regexp.new(queue_name_prefix)
30
+ @@queues.find_all {|q| q =~ filter}
31
+ end
32
+
33
+ def receive_messages(queue_url, maximum=1, visibility_timeout=nil)
34
+ if @@messages[queue_url] == []
35
+ []
36
+ else
37
+ visible = @@messages[queue_url].find_all {|m| m[:visible]}
38
+ @@log.puts "***** receiving #{visible.first[:id]} #{visible.first[:body]}"
39
+ [visible.first]
40
+ end
41
+ end
42
+
43
+ def send_message(queue_url, message_body, encode=false)
44
+ mid = "id-#{@@msg_seq}"
45
+ message = { :id => mid,
46
+ :body => message_body,
47
+ :visible => true}
48
+ @@msg_seq += 1
49
+ @@messages[queue_url] = [] unless @@messages[queue_url]
50
+ @@messages[queue_url] << message
51
+ @@log.puts "***** sending #{File.basename(queue_url)} #{mid} => #{message_body}"
52
+ true
53
+ end
54
+
55
+ def delete_message(queue_url, receipt_handle)
56
+ @@messages[queue_url] = @@messages[queue_url].reject {|m| m[:id] == receipt_handle}
57
+ true
58
+ end
59
+
60
+
61
+ def get_queue_attributes(queue_url, attribute='All')
62
+ @@attributes["ApproximateNumberOfMessages"] = @@messages[queue_url].size
63
+ @@attributes
64
+ end
65
+
66
+ ######################
67
+ # testing
68
+ def reset
69
+ @@log = STDOUT
70
+ @@queues = ["http://queue.amazonaws.com/A13T024T56MRDD/manual-manual-test",
71
+ "http://queue.amazonaws.com/A13T024T56MRDC/fib-status-test",
72
+ "http://queue.amazonaws.com/A13T024T56MRDC/fib-work-test",
73
+ "http://queue.amazonaws.com/A13T024T56MRDC/primes-status-test",
74
+ "http://queue.amazonaws.com/A13T024T56MRDC/primes-work-test"]
75
+
76
+ @@messages = {}
77
+ @@queues.each {|q| @@messages[q] = []}
78
+ @@attributes = {"ApproximateNumberOfMessages"=>0, "VisibilityTimeout"=>300}
79
+ @@msg_seq = 1
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,46 @@
1
+ require 'AWS/SimpleDB'
2
+
3
+ module MockAWS
4
+ class SimpleDB
5
+ def initialize(*params)
6
+ end
7
+
8
+ def list_domains(max_domains = 100)
9
+ []
10
+ end
11
+
12
+ def create_domain(domain_name)
13
+ true
14
+ end
15
+
16
+ def delete_domain(domain_name)
17
+ true
18
+ end
19
+
20
+ def put_attributes(domain_name, item_name, attributes, replace=false)
21
+ true
22
+ end
23
+
24
+ def delete_attributes(domain_name, item_name, attributes = {})
25
+ true
26
+ end
27
+
28
+ def get_attributes(domain_name, item_name, attribute_name = nil)
29
+ true
30
+ end
31
+
32
+ def query(domain_name, query_expression=nil, options={:fetch_all=>true})
33
+ true
34
+ end
35
+ ##############
36
+ # testing
37
+ def logger=(logger)
38
+ @@log = logger
39
+ end
40
+
41
+ def reset
42
+ @context = []
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,67 @@
1
+
2
+ # Used to mock Time
3
+
4
+ class Clock
5
+ attr_accessor :time
6
+ include Comparable
7
+
8
+ @@current_time = 0
9
+
10
+ def initialize(time)
11
+ @time = time
12
+ end
13
+
14
+ def Clock.at(time)
15
+ Clock.new(time)
16
+ end
17
+
18
+ def Clock.now
19
+ Clock.new(@@current_time)
20
+ end
21
+
22
+ def Clock.hour
23
+ (@@current_time/3600).floor
24
+ end
25
+
26
+ def to_s
27
+ @time.to_s
28
+ end
29
+
30
+ def +(time)
31
+ Clock.new(time + @time)
32
+ end
33
+
34
+ def -(clock)
35
+ @time - clock.time
36
+ end
37
+
38
+ def <=>(other)
39
+ @time <=> other.time
40
+ end
41
+
42
+ def strftime(format)
43
+ @time.to_s
44
+ end
45
+
46
+ def Clock.sleep(numeric = 0)
47
+ @@current_time += numeric
48
+ end
49
+
50
+ def Clock.xmlschema(tm)
51
+ Clock.new(Time.xmlschema(tm).to_i)
52
+ end
53
+
54
+ def Clock.parse(tm)
55
+ Clock.new(Time.parse(tm).to_i)
56
+ end
57
+
58
+ ##############
59
+ # testing
60
+ def Clock.reset
61
+ @@current_time = 0
62
+ end
63
+
64
+ def Clock.set(numeric)
65
+ @@current_time = numeric
66
+ end
67
+ end