auser-poolparty 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +12 -0
- data/Manifest +115 -0
- data/README.txt +140 -0
- data/Rakefile +27 -0
- data/bin/instance +61 -0
- data/bin/pool +62 -0
- data/config/cloud_master_takeover +17 -0
- data/config/create_proxy_ami.sh +582 -0
- data/config/haproxy.conf +29 -0
- data/config/heartbeat.conf +8 -0
- data/config/heartbeat_authkeys.conf +2 -0
- data/config/installers/ubuntu_install.sh +77 -0
- data/config/monit/haproxy.monit.conf +7 -0
- data/config/monit/nginx.monit.conf +0 -0
- data/config/monit.conf +9 -0
- data/config/nginx.conf +24 -0
- data/config/reconfigure_instances_script.sh +18 -0
- data/config/sample-config.yml +23 -0
- data/config/scp_instances_script.sh +12 -0
- data/lib/core/array.rb +13 -0
- data/lib/core/exception.rb +9 -0
- data/lib/core/float.rb +13 -0
- data/lib/core/hash.rb +11 -0
- data/lib/core/kernel.rb +12 -0
- data/lib/core/module.rb +22 -0
- data/lib/core/object.rb +18 -0
- data/lib/core/proc.rb +15 -0
- data/lib/core/string.rb +49 -0
- data/lib/core/time.rb +41 -0
- data/lib/modules/callback.rb +133 -0
- data/lib/modules/ec2_wrapper.rb +82 -0
- data/lib/modules/safe_instance.rb +31 -0
- data/lib/modules/vlad_override.rb +82 -0
- data/lib/poolparty/application.rb +170 -0
- data/lib/poolparty/init.rb +6 -0
- data/lib/poolparty/master.rb +329 -0
- data/lib/poolparty/monitors/cpu.rb +19 -0
- data/lib/poolparty/monitors/memory.rb +26 -0
- data/lib/poolparty/monitors/web.rb +23 -0
- data/lib/poolparty/monitors.rb +13 -0
- data/lib/poolparty/optioner.rb +16 -0
- data/lib/poolparty/plugin.rb +43 -0
- data/lib/poolparty/plugin_manager.rb +67 -0
- data/lib/poolparty/provider/packages/essential.rb +6 -0
- data/lib/poolparty/provider/packages/git.rb +4 -0
- data/lib/poolparty/provider/packages/haproxy.rb +20 -0
- data/lib/poolparty/provider/packages/heartbeat.rb +4 -0
- data/lib/poolparty/provider/packages/monit.rb +6 -0
- data/lib/poolparty/provider/packages/rsync.rb +4 -0
- data/lib/poolparty/provider/packages/ruby.rb +37 -0
- data/lib/poolparty/provider/packages/s3fuse.rb +11 -0
- data/lib/poolparty/provider/provider.rb +60 -0
- data/lib/poolparty/provider.rb +2 -0
- data/lib/poolparty/remote_instance.rb +216 -0
- data/lib/poolparty/remoter.rb +106 -0
- data/lib/poolparty/remoting.rb +112 -0
- data/lib/poolparty/scheduler.rb +103 -0
- data/lib/poolparty/tasks/cloud.rake +57 -0
- data/lib/poolparty/tasks/development.rake +38 -0
- data/lib/poolparty/tasks/ec2.rake +20 -0
- data/lib/poolparty/tasks/instance.rake +63 -0
- data/lib/poolparty/tasks/plugins.rake +30 -0
- data/lib/poolparty/tasks/server.rake +42 -0
- data/lib/poolparty/tasks.rb +29 -0
- data/lib/poolparty/tmp.rb +46 -0
- data/lib/poolparty.rb +105 -0
- data/lib/s3/s3_object_store_folders.rb +44 -0
- data/misc/basics_tutorial.txt +142 -0
- data/poolparty.gemspec +72 -0
- data/spec/application_spec.rb +39 -0
- data/spec/callback_spec.rb +194 -0
- data/spec/core_spec.rb +15 -0
- data/spec/helpers/ec2_mock.rb +44 -0
- data/spec/kernel_spec.rb +11 -0
- data/spec/master_spec.rb +203 -0
- data/spec/monitors/cpu_monitor_spec.rb +38 -0
- data/spec/monitors/memory_spec.rb +50 -0
- data/spec/monitors/misc_monitor_spec.rb +50 -0
- data/spec/monitors/web_spec.rb +39 -0
- data/spec/optioner_spec.rb +22 -0
- data/spec/plugin_manager_spec.rb +31 -0
- data/spec/plugin_spec.rb +101 -0
- data/spec/pool_binary_spec.rb +10 -0
- data/spec/poolparty_spec.rb +15 -0
- data/spec/provider_spec.rb +17 -0
- data/spec/remote_instance_spec.rb +149 -0
- data/spec/remoter_spec.rb +65 -0
- data/spec/remoting_spec.rb +84 -0
- data/spec/scheduler_spec.rb +75 -0
- data/spec/spec_helper.rb +39 -0
- data/spec/string_spec.rb +28 -0
- data/web/static/conf/nginx.conf +22 -0
- data/web/static/site/images/balloon.png +0 -0
- data/web/static/site/images/cb.png +0 -0
- data/web/static/site/images/clouds.png +0 -0
- data/web/static/site/images/railsconf_preso_img.png +0 -0
- data/web/static/site/index.html +71 -0
- data/web/static/site/javascripts/application.js +3 -0
- data/web/static/site/javascripts/corner.js +178 -0
- data/web/static/site/javascripts/jquery-1.2.6.pack.js +11 -0
- data/web/static/site/misc.html +42 -0
- data/web/static/site/storage/pool_party_presentation.pdf +0 -0
- data/web/static/site/stylesheets/application.css +100 -0
- data/web/static/site/stylesheets/reset.css +17 -0
- data/web/static/src/layouts/application.haml +25 -0
- data/web/static/src/pages/index.haml +25 -0
- data/web/static/src/pages/misc.haml +5 -0
- data/web/static/src/stylesheets/application.sass +100 -0
- metadata +260 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
require "vlad"
|
|
2
|
+
class Rake::RemoteTask < Rake::Task
|
|
3
|
+
def run command
|
|
4
|
+
cmd = [ssh_cmd, ssh_flags, target_host, command].compact
|
|
5
|
+
result = []
|
|
6
|
+
|
|
7
|
+
warn cmd.join(' ') if $TRACE
|
|
8
|
+
|
|
9
|
+
pid, inn, out, err = popen4(*cmd.join(" "))
|
|
10
|
+
|
|
11
|
+
inn.sync = true
|
|
12
|
+
streams = [out, err]
|
|
13
|
+
out_stream = {
|
|
14
|
+
out => $stdout,
|
|
15
|
+
err => $stderr,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
# Handle process termination ourselves
|
|
19
|
+
status = nil
|
|
20
|
+
Thread.start do
|
|
21
|
+
status = Process.waitpid2(pid).last
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
until streams.empty? do
|
|
25
|
+
# don't busy loop
|
|
26
|
+
selected, = select streams, nil, nil, 0.1
|
|
27
|
+
|
|
28
|
+
next if selected.nil? or selected.empty?
|
|
29
|
+
|
|
30
|
+
selected.each do |stream|
|
|
31
|
+
if stream.eof? then
|
|
32
|
+
streams.delete stream if status # we've quit, so no more writing
|
|
33
|
+
next
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
data = stream.readpartial(1024)
|
|
37
|
+
out_stream[stream].write data
|
|
38
|
+
|
|
39
|
+
if stream == err and data =~ /^Password:/ then
|
|
40
|
+
inn.puts sudo_password
|
|
41
|
+
data << "\n"
|
|
42
|
+
$stderr.write "\n"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
result << data
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
|
|
50
|
+
|
|
51
|
+
result.join
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def rsync local, remote
|
|
55
|
+
cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
|
|
56
|
+
|
|
57
|
+
success = system(*cmd.join(" "))
|
|
58
|
+
|
|
59
|
+
unless success then
|
|
60
|
+
raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
def set name, val = nil, &b
|
|
64
|
+
rt.set name, val, &b
|
|
65
|
+
end
|
|
66
|
+
def rt
|
|
67
|
+
@rt ||= Rake::RemoteTask
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def target_hosts
|
|
71
|
+
if hosts = ENV["HOSTS"] then
|
|
72
|
+
hosts.strip.gsub(/\s+/, '').split(",")
|
|
73
|
+
elsif options[:single]
|
|
74
|
+
@roles = {}; @roles[:app] = {}
|
|
75
|
+
@roles[:app][options[:single]] = options[:single]
|
|
76
|
+
roles = Rake::RemoteTask.hosts_for(@roles)
|
|
77
|
+
else
|
|
78
|
+
roles = options[:roles]
|
|
79
|
+
roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
=begin rdoc
|
|
2
|
+
Application
|
|
3
|
+
This handles user interaction
|
|
4
|
+
=end
|
|
5
|
+
module PoolParty
|
|
6
|
+
class Application
|
|
7
|
+
class << self
|
|
8
|
+
attr_accessor :verbose, :options
|
|
9
|
+
|
|
10
|
+
# The application options
|
|
11
|
+
def options(opts={})
|
|
12
|
+
@options ||= make_options(opts)
|
|
13
|
+
end
|
|
14
|
+
# Make the options with the config_file overrides included
|
|
15
|
+
# Default config file assumed to be at config/config.yml
|
|
16
|
+
def make_options(opts={})
|
|
17
|
+
loading_options = opts.delete(:optsparse) || {}
|
|
18
|
+
loading_options.merge!( {:argv => opts.delete(:argv)} )
|
|
19
|
+
|
|
20
|
+
load_options!(loading_options)
|
|
21
|
+
# default_options.merge!(opts)
|
|
22
|
+
# If the config_file options are specified and not empty
|
|
23
|
+
unless default_options[:config_file].nil? || default_options[:config_file].empty?
|
|
24
|
+
require "yaml"
|
|
25
|
+
# Try loading the file if it exists
|
|
26
|
+
filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
|
|
27
|
+
default_options.merge!( YAML.load(filedata) ) if filedata # We want the command-line to overwrite the config file
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
OpenStruct.new(default_options.merge(opts))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Load options via commandline
|
|
34
|
+
def load_options!(opts={})
|
|
35
|
+
require 'optparse'
|
|
36
|
+
OptionParser.new do |op|
|
|
37
|
+
op.banner = opts[:banner] if opts[:banner]
|
|
38
|
+
op.on('-A key', '--access-key key', "Ec2 access key (ENV['ACCESS_KEY'])") { |key| default_options[:access_key] = key }
|
|
39
|
+
op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['SECRET_ACCESS_KEY'])") { |key| default_options[:secret_access_key] = key }
|
|
40
|
+
op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-4a46a323')") {|id| default_options[:ami] = id }
|
|
41
|
+
op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
|
|
42
|
+
op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }
|
|
43
|
+
op.on('-D ec2 directory', '--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
|
|
44
|
+
op.on('-r names', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
|
|
45
|
+
op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
|
|
46
|
+
op.on('-l plugin_dir', '--plugin-dir dir', "Plugin directory (default: '')") {|file| default_options[:plugin_dir] = file }
|
|
47
|
+
op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
|
|
48
|
+
op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }
|
|
49
|
+
op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
|
|
50
|
+
op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }
|
|
51
|
+
op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
|
|
52
|
+
op.on('-a address', '--public-ip address', "Associate this public address with the master node") {|s| default_options[:public_ip] = s}
|
|
53
|
+
op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
|
|
54
|
+
op.on('-n name', '--name name', "Application name") {|n| default_options[:app_name] = n}
|
|
55
|
+
op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
|
|
56
|
+
op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
|
|
57
|
+
op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
|
|
58
|
+
op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = true}
|
|
59
|
+
op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
|
|
60
|
+
op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x.to_i}
|
|
61
|
+
|
|
62
|
+
op.on_tail("-V", "Show version") do
|
|
63
|
+
puts Application.version
|
|
64
|
+
exit
|
|
65
|
+
end
|
|
66
|
+
op.on_tail("-h", "-?", "Show this message") do
|
|
67
|
+
puts op
|
|
68
|
+
exit
|
|
69
|
+
end
|
|
70
|
+
end.parse!(opts[:argv] ? opts.delete(:argv) : ARGV.dup)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Basic default options
|
|
74
|
+
# All can be overridden by the command line
|
|
75
|
+
# or in a config.yml file
|
|
76
|
+
def default_options
|
|
77
|
+
@default_options ||= {
|
|
78
|
+
:app_name => "application_name",
|
|
79
|
+
:host_port => 80,
|
|
80
|
+
:client_port => 8001,
|
|
81
|
+
:environment => 'development',
|
|
82
|
+
:verbose => false,
|
|
83
|
+
:logging => true,
|
|
84
|
+
:size => "m1.small",
|
|
85
|
+
:polling_time => "30.seconds",
|
|
86
|
+
:user_data => "",
|
|
87
|
+
:heavy_load => 0.80,
|
|
88
|
+
:light_load => 0.15,
|
|
89
|
+
:minimum_instances => 2,
|
|
90
|
+
:maximum_instances => 4,
|
|
91
|
+
:public_ip => "",
|
|
92
|
+
:access_key => ENV["AWS_ACCESS_KEY_ID"],
|
|
93
|
+
:secret_access_key => ENV["AWS_SECRET_ACCESS_ID"],
|
|
94
|
+
:config_file => if ENV["CONFIG_FILE"] && !ENV["CONFIG_FILE"].empty?
|
|
95
|
+
ENV["CONFIG_FILE"]
|
|
96
|
+
elsif File.file?("config/config.yml")
|
|
97
|
+
"config/config.yml"
|
|
98
|
+
else
|
|
99
|
+
nil
|
|
100
|
+
end,
|
|
101
|
+
:username => "root",
|
|
102
|
+
:ec2_dir => ENV["EC2_HOME"],
|
|
103
|
+
:keypair => ENV["KEYPAIR_NAME"],
|
|
104
|
+
:ami => 'ami-4a46a323',
|
|
105
|
+
:shared_bucket => "",
|
|
106
|
+
:services => "",
|
|
107
|
+
:expand_when => "web_usage < 1.5\n memory > 0.85",
|
|
108
|
+
:contract_when => "cpu < 0.20\n memory < 0.10",
|
|
109
|
+
:os => "ubuntu",
|
|
110
|
+
:plugin_dir => "vendor",
|
|
111
|
+
:install_on_load => true
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
# Services monitored by Heartbeat
|
|
115
|
+
# Always at least monitors haproxy
|
|
116
|
+
def managed_services
|
|
117
|
+
"#{services}"
|
|
118
|
+
end
|
|
119
|
+
def master_managed_services
|
|
120
|
+
"cloud_master_takeover #{services}"
|
|
121
|
+
end
|
|
122
|
+
def launching_user_data
|
|
123
|
+
{:polling_time => polling_time}.to_yaml
|
|
124
|
+
end
|
|
125
|
+
# Keypair path
|
|
126
|
+
# Idiom:
|
|
127
|
+
# /Users/username/.ec2/id_rsa-name
|
|
128
|
+
def keypair_path
|
|
129
|
+
"#{ec2_dir}/id_rsa#{keypair ? "-#{keypair}" : "" }"
|
|
130
|
+
end
|
|
131
|
+
# Are we in development or test mode
|
|
132
|
+
%w(development production test).each do |env|
|
|
133
|
+
eval <<-EOE
|
|
134
|
+
def #{env}?
|
|
135
|
+
environment == '#{env}'
|
|
136
|
+
end
|
|
137
|
+
EOE
|
|
138
|
+
end
|
|
139
|
+
def environment=(env)
|
|
140
|
+
environment = env
|
|
141
|
+
end
|
|
142
|
+
def maintain_pid_path
|
|
143
|
+
"/var/run/pool_maintain.pid"
|
|
144
|
+
end
|
|
145
|
+
%w(scp_instances_script reconfigure_instances_script).each do |file|
|
|
146
|
+
define_method "sh_#{file}" do
|
|
147
|
+
File.join(File.dirname(__FILE__), "../..", "config", "#{file}.sh")
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
# Standard configuration files
|
|
151
|
+
%w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
|
|
152
|
+
define_method "#{file}_config_file" do
|
|
153
|
+
File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
def version
|
|
157
|
+
PoolParty::Version::STRING
|
|
158
|
+
end
|
|
159
|
+
def install_on_load?
|
|
160
|
+
options.install_on_load == true
|
|
161
|
+
end
|
|
162
|
+
# Call the options from the Application
|
|
163
|
+
def method_missing(m,*args)
|
|
164
|
+
options.methods.include?("#{m}") ? options.send(m,args) : super
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
end
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
=begin rdoc
|
|
2
|
+
The basic master for PoolParty
|
|
3
|
+
=end
|
|
4
|
+
module PoolParty
|
|
5
|
+
class Master < Remoting
|
|
6
|
+
include Aska
|
|
7
|
+
include Callbacks
|
|
8
|
+
include Monitors
|
|
9
|
+
include Remoter
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
super
|
|
13
|
+
|
|
14
|
+
self.class.send :rules, :contract_when, Application.options.contract_when
|
|
15
|
+
self.class.send :rules, :expand_when, Application.options.expand_when
|
|
16
|
+
end
|
|
17
|
+
# Start the cloud
|
|
18
|
+
def start_cloud!
|
|
19
|
+
start!
|
|
20
|
+
end
|
|
21
|
+
alias_method :start_cloud, :start_cloud!
|
|
22
|
+
# Start the cloud, which launches the minimum_instances
|
|
23
|
+
def start!
|
|
24
|
+
message "Launching minimum_instances"
|
|
25
|
+
launch_minimum_instances
|
|
26
|
+
message "Waiting for master to boot up"
|
|
27
|
+
reset!
|
|
28
|
+
while !number_of_pending_instances.zero?
|
|
29
|
+
wait "2.seconds" unless Application.test?
|
|
30
|
+
waited = true
|
|
31
|
+
reset!
|
|
32
|
+
end
|
|
33
|
+
unless Application.test? || waited.nil?
|
|
34
|
+
message "Give some time for the instance ssh to start up"
|
|
35
|
+
wait "15.seconds"
|
|
36
|
+
end
|
|
37
|
+
install_cloud if Application.install_on_load?
|
|
38
|
+
configure_cloud
|
|
39
|
+
end
|
|
40
|
+
alias_method :start, :start!
|
|
41
|
+
def configure_cloud
|
|
42
|
+
message "Configuring master"
|
|
43
|
+
master = get_node 0
|
|
44
|
+
master.configure
|
|
45
|
+
end
|
|
46
|
+
def install_cloud
|
|
47
|
+
Master.with_nodes do |node|
|
|
48
|
+
node.login_once
|
|
49
|
+
end
|
|
50
|
+
Provider.install_poolparty(nodes.collect {|a| a.ip })
|
|
51
|
+
end
|
|
52
|
+
# Launch the minimum number of instances.
|
|
53
|
+
def launch_minimum_instances
|
|
54
|
+
request_launch_new_instances(Application.minimum_instances - number_of_pending_and_running_instances)
|
|
55
|
+
nodes
|
|
56
|
+
end
|
|
57
|
+
# Start monitoring the cloud with the threaded loop
|
|
58
|
+
def start_monitor!
|
|
59
|
+
begin
|
|
60
|
+
trap("INT") do
|
|
61
|
+
on_exit
|
|
62
|
+
exit
|
|
63
|
+
end
|
|
64
|
+
# Daemonize only if we are not in the test environment
|
|
65
|
+
run_thread_loop(:daemonize => !Application.test?) do
|
|
66
|
+
add_task {launch_minimum_instances} # If the base instances go down...
|
|
67
|
+
add_task {reconfigure_cloud_when_necessary}
|
|
68
|
+
add_task {scale_cloud!}
|
|
69
|
+
add_task {check_stats}
|
|
70
|
+
end
|
|
71
|
+
rescue Exception => e
|
|
72
|
+
puts "There was an error: #{e.nice_message}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
alias_method :start_monitor, :start_monitor!
|
|
76
|
+
def user_tasks
|
|
77
|
+
puts "in user_tasks"
|
|
78
|
+
end
|
|
79
|
+
# Sole purpose to check the stats, mainly in a plugin
|
|
80
|
+
def check_stats
|
|
81
|
+
end
|
|
82
|
+
# Add an instance if the cloud needs one ore terminate one if necessary
|
|
83
|
+
def scale_cloud!
|
|
84
|
+
add_instance_if_load_is_high
|
|
85
|
+
terminate_instance_if_load_is_low
|
|
86
|
+
end
|
|
87
|
+
alias_method :scale_cloud, :scale_cloud!
|
|
88
|
+
# Tough method:
|
|
89
|
+
# We need to make sure that all the instances have the required software installed
|
|
90
|
+
# This is a basic check against the local store of the instances that have the
|
|
91
|
+
# stack installed.
|
|
92
|
+
def reconfigure_cloud_when_necessary
|
|
93
|
+
reconfigure_running_instances if number_of_unconfigured_nodes > 0
|
|
94
|
+
end
|
|
95
|
+
alias_method :reconfiguration, :reconfigure_cloud_when_necessary
|
|
96
|
+
def number_of_unconfigured_nodes
|
|
97
|
+
nodes.reject {|a| a.stack_installed? }.size
|
|
98
|
+
end
|
|
99
|
+
def grow_by_one
|
|
100
|
+
request_launch_new_instance
|
|
101
|
+
self.class.get_master.configure
|
|
102
|
+
end
|
|
103
|
+
def shrink_by_one
|
|
104
|
+
node = nodes.reject {|a| a.master? }[-1]
|
|
105
|
+
request_termination_of_instance(node.instance_id) if node
|
|
106
|
+
end
|
|
107
|
+
# Add an instance if the load is high
|
|
108
|
+
def add_instance_if_load_is_high
|
|
109
|
+
request_launch_new_instance if expand?
|
|
110
|
+
end
|
|
111
|
+
alias_method :add_instance, :add_instance_if_load_is_high
|
|
112
|
+
# Teardown an instance if the load is pretty low
|
|
113
|
+
def terminate_instance_if_load_is_low
|
|
114
|
+
if contract?
|
|
115
|
+
shrink_by_one
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
alias_method :terminate_instance, :terminate_instance_if_load_is_low
|
|
119
|
+
# FOR MONITORING
|
|
120
|
+
def contract?
|
|
121
|
+
valid_rules?(:contract_when)
|
|
122
|
+
end
|
|
123
|
+
def expand?
|
|
124
|
+
valid_rules?(:expand_when)
|
|
125
|
+
end
|
|
126
|
+
# Restart the running instances services with monit on all the nodes
|
|
127
|
+
def restart_running_instances_services
|
|
128
|
+
nodes.each do |node|
|
|
129
|
+
node.restart_with_monit
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
# Reconfigure the running instances
|
|
133
|
+
# Since we are using vlad, running configure on one of the instances
|
|
134
|
+
# should configure all of the instances. We set the hosts in this file
|
|
135
|
+
def reconfigure_running_instances
|
|
136
|
+
# nodes.each do |node|
|
|
137
|
+
# node.configure if node.status =~ /running/
|
|
138
|
+
# end
|
|
139
|
+
master = get_node(0)
|
|
140
|
+
master.configure
|
|
141
|
+
end
|
|
142
|
+
# Build the basic haproxy config file from the config file in the config directory and return a tempfile
|
|
143
|
+
def build_haproxy_file
|
|
144
|
+
servers=<<-EOS
|
|
145
|
+
#{nodes.collect {|node| node.haproxy_entry}.join("\n")}
|
|
146
|
+
EOS
|
|
147
|
+
open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
|
|
148
|
+
end
|
|
149
|
+
# Build the hosts file and return a tempfile
|
|
150
|
+
def build_hosts_file
|
|
151
|
+
write_to_temp_file(nodes.collect {|a| a.hosts_entry }.join("\n"))
|
|
152
|
+
end
|
|
153
|
+
# Build host file for a specific node
|
|
154
|
+
def build_hosts_file_for(n)
|
|
155
|
+
servers=<<-EOS
|
|
156
|
+
#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}
|
|
157
|
+
EOS
|
|
158
|
+
servers
|
|
159
|
+
end
|
|
160
|
+
# Build the basic auth file for the heartbeat
|
|
161
|
+
def build_heartbeat_authkeys_file
|
|
162
|
+
write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read)
|
|
163
|
+
end
|
|
164
|
+
# Build heartbeat config file
|
|
165
|
+
def build_heartbeat_config_file_for(node)
|
|
166
|
+
servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}"
|
|
167
|
+
open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers}
|
|
168
|
+
end
|
|
169
|
+
# Return a list of the nodes and cache them
|
|
170
|
+
def nodes
|
|
171
|
+
@nodes ||= list_of_nonterminated_instances.collect_with_index do |inst, i|
|
|
172
|
+
RemoteInstance.new(inst.merge({:number => i}))
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
# Get the node at the specific index from the cached nodes
|
|
176
|
+
def get_node(i=0)
|
|
177
|
+
nodes.select {|a| a.number == i.to_i}.first
|
|
178
|
+
end
|
|
179
|
+
# Get the next node in sequence, so we can configure heartbeat to monitor the next node
|
|
180
|
+
def get_next_node(node)
|
|
181
|
+
i = node.number + 1
|
|
182
|
+
i = 0 if i >= (nodes.size - 1)
|
|
183
|
+
get_node(i)
|
|
184
|
+
end
|
|
185
|
+
# On exit command
|
|
186
|
+
def on_exit
|
|
187
|
+
end
|
|
188
|
+
# List the clouds
|
|
189
|
+
def list
|
|
190
|
+
if number_of_pending_and_running_instances > 0
|
|
191
|
+
out = "-- CLOUD (#{number_of_pending_and_running_instances})--\n"
|
|
192
|
+
out << nodes.collect {|node| node.description }.join("\n")
|
|
193
|
+
else
|
|
194
|
+
out = "Cloud is not running"
|
|
195
|
+
end
|
|
196
|
+
out
|
|
197
|
+
end
|
|
198
|
+
# Reset and clear the caches
|
|
199
|
+
def reset!
|
|
200
|
+
@cached_descriptions = nil
|
|
201
|
+
@nodes = nil
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
class << self
|
|
205
|
+
include PoolParty
|
|
206
|
+
|
|
207
|
+
def with_nodes(&block)
|
|
208
|
+
new.nodes.each &block
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def collect_nodes(&block)
|
|
212
|
+
new.nodes.collect &block
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def requires_heartbeat?
|
|
216
|
+
new.nodes.size > 1
|
|
217
|
+
end
|
|
218
|
+
def is_master_responding?
|
|
219
|
+
`ping -c1 -t5 #{get_master.ip}`
|
|
220
|
+
end
|
|
221
|
+
def get_master
|
|
222
|
+
new.nodes[0]
|
|
223
|
+
end
|
|
224
|
+
def get_next_node(node)
|
|
225
|
+
new.get_next_node(node)
|
|
226
|
+
end
|
|
227
|
+
# Build a heartbeat_config_file from the config file in the config directory and return a tempfile
|
|
228
|
+
def build_heartbeat_config_file_for(node)
|
|
229
|
+
return nil unless node
|
|
230
|
+
new.build_heartbeat_config_file_for(node)
|
|
231
|
+
end
|
|
232
|
+
# Build a heartbeat resources file from the config directory and return a tempfile
|
|
233
|
+
def build_heartbeat_resources_file_for(node)
|
|
234
|
+
return nil unless node
|
|
235
|
+
"#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}"
|
|
236
|
+
end
|
|
237
|
+
# Build hosts files for a specific node
|
|
238
|
+
def build_hosts_file_for(node)
|
|
239
|
+
new.build_hosts_file_for(node)
|
|
240
|
+
end
|
|
241
|
+
# Build the scp script for the specific node
|
|
242
|
+
def build_scp_instances_script_for(node)
|
|
243
|
+
authkeys_file = write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read.strip)
|
|
244
|
+
if Master.requires_heartbeat?
|
|
245
|
+
ha_d_file = Master.build_heartbeat_config_file_for(node)
|
|
246
|
+
haresources_file = Master.build_heartbeat_resources_file_for(node)
|
|
247
|
+
end
|
|
248
|
+
haproxy_file = Master.build_haproxy_file
|
|
249
|
+
hosts_file = Master.build_hosts_file_for(node)
|
|
250
|
+
|
|
251
|
+
str = open(Application.sh_scp_instances_script).read.strip ^ {
|
|
252
|
+
:cloud_master_takeover => "#{node.scp_string("#{root_dir}/config/cloud_master_takeover", "/etc/ha.d/resource.d/", :dir => "/etc/ha.d/resource.d")}",
|
|
253
|
+
:config_file => "#{node.scp_string(Application.config_file, "~/.config")}",
|
|
254
|
+
:authkeys => "#{node.scp_string(authkeys_file.path, "/etc/ha.d/authkeys", :dir => "/etc/ha.d/")}",
|
|
255
|
+
:resources => "#{node.scp_string("#{root_dir}/config/resource.d/*", "/etc/ha.d/resource.d/", {:switches => "-r"})}",
|
|
256
|
+
:monitrc => "#{node.scp_string(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")}",
|
|
257
|
+
:monit_d => "#{node.scp_string("#{File.dirname(Application.monit_config_file)}/monit/*", "/etc/monit.d/", {:switches => "-r", :dir => "/etc/monit.d/"})}",
|
|
258
|
+
:haproxy => "#{node.scp_string(haproxy_file, "/etc/haproxy.cfg")}",
|
|
259
|
+
|
|
260
|
+
:ha_d => Master.requires_heartbeat? ? "#{node.scp_string(ha_d_file, "/etc/ha.d/ha.cf")}" : "",
|
|
261
|
+
:haresources => Master.requires_heartbeat? ? "#{node.scp_string(haresources_file, "/etc/ha.d/ha.cf")}" : "",
|
|
262
|
+
|
|
263
|
+
:hosts => "#{node.scp_string(hosts_file, "/etc/hosts")}"
|
|
264
|
+
}
|
|
265
|
+
write_to_temp_file(str)
|
|
266
|
+
end
|
|
267
|
+
# Build basic configuration script for the node
|
|
268
|
+
def build_reconfigure_instances_script_for(node)
|
|
269
|
+
str = open(Application.sh_reconfigure_instances_script).read.strip ^ {
|
|
270
|
+
:config_master => "#{node.update_plugin_string}",
|
|
271
|
+
:start_pool_maintain => "pool maintain -c ~/.config -l ~/plugins",
|
|
272
|
+
:set_hostname => "hostname -v #{node.name}",
|
|
273
|
+
:start_s3fs => "/usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data"
|
|
274
|
+
}
|
|
275
|
+
write_to_temp_file(str)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def set_hosts(c, remotetask=nil)
|
|
279
|
+
unless remotetask.nil?
|
|
280
|
+
rt = remotetask
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
ssh_location = `which ssh`.gsub(/\n/, '')
|
|
284
|
+
rsync_location = `which rsync`.gsub(/\n/, '')
|
|
285
|
+
rt.set :user, Application.username
|
|
286
|
+
# rt.set :domain, "#{Application.user}@#{ip}"
|
|
287
|
+
rt.set :application, Application.app_name
|
|
288
|
+
rt.set :ssh_flags, "-i #{Application.keypair_path} -o StrictHostKeyChecking=no"
|
|
289
|
+
rt.set :rsync_flags , ['-azP', '--delete', "-e '#{ssh_location} -l #{Application.username} -i #{Application.keypair_path} -o StrictHostKeyChecking=no'"]
|
|
290
|
+
|
|
291
|
+
master = get_master
|
|
292
|
+
rt.set :domain, "#{master.ip}" if master
|
|
293
|
+
Master.with_nodes { |node|
|
|
294
|
+
rt.host "#{Application.username}@#{node.ip}",:app if node.status =~ /running/
|
|
295
|
+
}
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def ssh_configure_string_for(node)
|
|
299
|
+
cmd=<<-EOC
|
|
300
|
+
#{node.update_plugin_string(node)}
|
|
301
|
+
pool maintain -c ~/.config -l #{PoolParty.plugin_dir}
|
|
302
|
+
hostname -v #{node.name}
|
|
303
|
+
/usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
|
|
304
|
+
EOC
|
|
305
|
+
end
|
|
306
|
+
def build_haproxy_file
|
|
307
|
+
servers=<<-EOS
|
|
308
|
+
#{collect_nodes {|node| node.haproxy_entry}.join("\n")}
|
|
309
|
+
EOS
|
|
310
|
+
open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
|
|
311
|
+
end
|
|
312
|
+
# Write a temp file with the content str
|
|
313
|
+
def write_to_temp_file(str="")
|
|
314
|
+
tempfile = Tempfile.new("pool-party-#{rand(1000)}-#{rand(1000)}")
|
|
315
|
+
tempfile.print(str)
|
|
316
|
+
tempfile.flush
|
|
317
|
+
tempfile
|
|
318
|
+
end
|
|
319
|
+
def with_temp_file(str="", &block)
|
|
320
|
+
Tempfile.open "pool-party-#{rand(10000)}" do |fp|
|
|
321
|
+
fp.puts str
|
|
322
|
+
fp.flush
|
|
323
|
+
block.call(fp)
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
end
|
|
329
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
=begin rdoc
|
|
2
|
+
Basic monitor on the cpu stats
|
|
3
|
+
=end
|
|
4
|
+
module Cpu
|
|
5
|
+
module Master
|
|
6
|
+
def cpu
|
|
7
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i+=a.cpu } / nodes.size : 0.0
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
module Remote
|
|
12
|
+
def cpu
|
|
13
|
+
ssh("uptime").split(/\s+/)[-3].to_f rescue 0.0
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
PoolParty.register_monitor Cpu
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
=begin rdoc
|
|
2
|
+
Basic monitor on the cpu stats
|
|
3
|
+
=end
|
|
4
|
+
module Memory
|
|
5
|
+
module Master
|
|
6
|
+
# Get the average memory usage over the cloud
|
|
7
|
+
def memory
|
|
8
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.memory } / nodes.size : 0.0
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
module Remote
|
|
13
|
+
def memory
|
|
14
|
+
str = ssh("free -m | grep -i mem")
|
|
15
|
+
total_memory = str.split[1].to_f
|
|
16
|
+
used_memory = str.split[2].to_f
|
|
17
|
+
|
|
18
|
+
used_memory / total_memory
|
|
19
|
+
rescue
|
|
20
|
+
0.0
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
PoolParty.register_monitor Memory
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
=begin rdoc
|
|
2
|
+
Basic monitor on the cpu stats
|
|
3
|
+
=end
|
|
4
|
+
module Web
|
|
5
|
+
module Master
|
|
6
|
+
# Get the average web request capabilities over the cloud
|
|
7
|
+
def web
|
|
8
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.web } / nodes.size : 0.0
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
module Remote
|
|
13
|
+
def web
|
|
14
|
+
str = ssh("httperf --server localhost --port #{Application.client_port} --num-conn 3 --timeout 5 | grep 'Request rate'")
|
|
15
|
+
str[/[.]* ([\d]*\.[\d]*) [.]*/, 0].chomp.to_f
|
|
16
|
+
rescue
|
|
17
|
+
0.0
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
PoolParty.register_monitor Web
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module PoolParty
|
|
2
|
+
class Optioner
|
|
3
|
+
# Parse the command line options for options without a switch
|
|
4
|
+
def self.parse(argv, safe=[])
|
|
5
|
+
args = []
|
|
6
|
+
argv.each_with_index do |arg,i|
|
|
7
|
+
unless arg.index("-")
|
|
8
|
+
args << arg
|
|
9
|
+
else
|
|
10
|
+
argv.delete_at(i+1) unless safe.include?(arg)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
args
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|