auser-poolparty 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +12 -0
- data/Manifest +115 -0
- data/README.txt +140 -0
- data/Rakefile +27 -0
- data/bin/instance +61 -0
- data/bin/pool +62 -0
- data/config/cloud_master_takeover +17 -0
- data/config/create_proxy_ami.sh +582 -0
- data/config/haproxy.conf +29 -0
- data/config/heartbeat.conf +8 -0
- data/config/heartbeat_authkeys.conf +2 -0
- data/config/installers/ubuntu_install.sh +77 -0
- data/config/monit/haproxy.monit.conf +7 -0
- data/config/monit/nginx.monit.conf +0 -0
- data/config/monit.conf +9 -0
- data/config/nginx.conf +24 -0
- data/config/reconfigure_instances_script.sh +18 -0
- data/config/sample-config.yml +23 -0
- data/config/scp_instances_script.sh +12 -0
- data/lib/core/array.rb +13 -0
- data/lib/core/exception.rb +9 -0
- data/lib/core/float.rb +13 -0
- data/lib/core/hash.rb +11 -0
- data/lib/core/kernel.rb +12 -0
- data/lib/core/module.rb +22 -0
- data/lib/core/object.rb +18 -0
- data/lib/core/proc.rb +15 -0
- data/lib/core/string.rb +49 -0
- data/lib/core/time.rb +41 -0
- data/lib/modules/callback.rb +133 -0
- data/lib/modules/ec2_wrapper.rb +82 -0
- data/lib/modules/safe_instance.rb +31 -0
- data/lib/modules/vlad_override.rb +82 -0
- data/lib/poolparty/application.rb +170 -0
- data/lib/poolparty/init.rb +6 -0
- data/lib/poolparty/master.rb +329 -0
- data/lib/poolparty/monitors/cpu.rb +19 -0
- data/lib/poolparty/monitors/memory.rb +26 -0
- data/lib/poolparty/monitors/web.rb +23 -0
- data/lib/poolparty/monitors.rb +13 -0
- data/lib/poolparty/optioner.rb +16 -0
- data/lib/poolparty/plugin.rb +43 -0
- data/lib/poolparty/plugin_manager.rb +67 -0
- data/lib/poolparty/provider/packages/essential.rb +6 -0
- data/lib/poolparty/provider/packages/git.rb +4 -0
- data/lib/poolparty/provider/packages/haproxy.rb +20 -0
- data/lib/poolparty/provider/packages/heartbeat.rb +4 -0
- data/lib/poolparty/provider/packages/monit.rb +6 -0
- data/lib/poolparty/provider/packages/rsync.rb +4 -0
- data/lib/poolparty/provider/packages/ruby.rb +37 -0
- data/lib/poolparty/provider/packages/s3fuse.rb +11 -0
- data/lib/poolparty/provider/provider.rb +60 -0
- data/lib/poolparty/provider.rb +2 -0
- data/lib/poolparty/remote_instance.rb +216 -0
- data/lib/poolparty/remoter.rb +106 -0
- data/lib/poolparty/remoting.rb +112 -0
- data/lib/poolparty/scheduler.rb +103 -0
- data/lib/poolparty/tasks/cloud.rake +57 -0
- data/lib/poolparty/tasks/development.rake +38 -0
- data/lib/poolparty/tasks/ec2.rake +20 -0
- data/lib/poolparty/tasks/instance.rake +63 -0
- data/lib/poolparty/tasks/plugins.rake +30 -0
- data/lib/poolparty/tasks/server.rake +42 -0
- data/lib/poolparty/tasks.rb +29 -0
- data/lib/poolparty/tmp.rb +46 -0
- data/lib/poolparty.rb +105 -0
- data/lib/s3/s3_object_store_folders.rb +44 -0
- data/misc/basics_tutorial.txt +142 -0
- data/poolparty.gemspec +72 -0
- data/spec/application_spec.rb +39 -0
- data/spec/callback_spec.rb +194 -0
- data/spec/core_spec.rb +15 -0
- data/spec/helpers/ec2_mock.rb +44 -0
- data/spec/kernel_spec.rb +11 -0
- data/spec/master_spec.rb +203 -0
- data/spec/monitors/cpu_monitor_spec.rb +38 -0
- data/spec/monitors/memory_spec.rb +50 -0
- data/spec/monitors/misc_monitor_spec.rb +50 -0
- data/spec/monitors/web_spec.rb +39 -0
- data/spec/optioner_spec.rb +22 -0
- data/spec/plugin_manager_spec.rb +31 -0
- data/spec/plugin_spec.rb +101 -0
- data/spec/pool_binary_spec.rb +10 -0
- data/spec/poolparty_spec.rb +15 -0
- data/spec/provider_spec.rb +17 -0
- data/spec/remote_instance_spec.rb +149 -0
- data/spec/remoter_spec.rb +65 -0
- data/spec/remoting_spec.rb +84 -0
- data/spec/scheduler_spec.rb +75 -0
- data/spec/spec_helper.rb +39 -0
- data/spec/string_spec.rb +28 -0
- data/web/static/conf/nginx.conf +22 -0
- data/web/static/site/images/balloon.png +0 -0
- data/web/static/site/images/cb.png +0 -0
- data/web/static/site/images/clouds.png +0 -0
- data/web/static/site/images/railsconf_preso_img.png +0 -0
- data/web/static/site/index.html +71 -0
- data/web/static/site/javascripts/application.js +3 -0
- data/web/static/site/javascripts/corner.js +178 -0
- data/web/static/site/javascripts/jquery-1.2.6.pack.js +11 -0
- data/web/static/site/misc.html +42 -0
- data/web/static/site/storage/pool_party_presentation.pdf +0 -0
- data/web/static/site/stylesheets/application.css +100 -0
- data/web/static/site/stylesheets/reset.css +17 -0
- data/web/static/src/layouts/application.haml +25 -0
- data/web/static/src/pages/index.haml +25 -0
- data/web/static/src/pages/misc.haml +5 -0
- data/web/static/src/stylesheets/application.sass +100 -0
- metadata +260 -0
@@ -0,0 +1,82 @@
|
|
1
|
+
require "vlad"
|
2
|
+
class Rake::RemoteTask < Rake::Task
|
3
|
+
def run command
|
4
|
+
cmd = [ssh_cmd, ssh_flags, target_host, command].compact
|
5
|
+
result = []
|
6
|
+
|
7
|
+
warn cmd.join(' ') if $TRACE
|
8
|
+
|
9
|
+
pid, inn, out, err = popen4(*cmd.join(" "))
|
10
|
+
|
11
|
+
inn.sync = true
|
12
|
+
streams = [out, err]
|
13
|
+
out_stream = {
|
14
|
+
out => $stdout,
|
15
|
+
err => $stderr,
|
16
|
+
}
|
17
|
+
|
18
|
+
# Handle process termination ourselves
|
19
|
+
status = nil
|
20
|
+
Thread.start do
|
21
|
+
status = Process.waitpid2(pid).last
|
22
|
+
end
|
23
|
+
|
24
|
+
until streams.empty? do
|
25
|
+
# don't busy loop
|
26
|
+
selected, = select streams, nil, nil, 0.1
|
27
|
+
|
28
|
+
next if selected.nil? or selected.empty?
|
29
|
+
|
30
|
+
selected.each do |stream|
|
31
|
+
if stream.eof? then
|
32
|
+
streams.delete stream if status # we've quit, so no more writing
|
33
|
+
next
|
34
|
+
end
|
35
|
+
|
36
|
+
data = stream.readpartial(1024)
|
37
|
+
out_stream[stream].write data
|
38
|
+
|
39
|
+
if stream == err and data =~ /^Password:/ then
|
40
|
+
inn.puts sudo_password
|
41
|
+
data << "\n"
|
42
|
+
$stderr.write "\n"
|
43
|
+
end
|
44
|
+
|
45
|
+
result << data
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
|
50
|
+
|
51
|
+
result.join
|
52
|
+
end
|
53
|
+
|
54
|
+
def rsync local, remote
|
55
|
+
cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
|
56
|
+
|
57
|
+
success = system(*cmd.join(" "))
|
58
|
+
|
59
|
+
unless success then
|
60
|
+
raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
def set name, val = nil, &b
|
64
|
+
rt.set name, val, &b
|
65
|
+
end
|
66
|
+
def rt
|
67
|
+
@rt ||= Rake::RemoteTask
|
68
|
+
end
|
69
|
+
|
70
|
+
def target_hosts
|
71
|
+
if hosts = ENV["HOSTS"] then
|
72
|
+
hosts.strip.gsub(/\s+/, '').split(",")
|
73
|
+
elsif options[:single]
|
74
|
+
@roles = {}; @roles[:app] = {}
|
75
|
+
@roles[:app][options[:single]] = options[:single]
|
76
|
+
roles = Rake::RemoteTask.hosts_for(@roles)
|
77
|
+
else
|
78
|
+
roles = options[:roles]
|
79
|
+
roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,170 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
Application
|
3
|
+
This handles user interaction
|
4
|
+
=end
|
5
|
+
module PoolParty
|
6
|
+
class Application
|
7
|
+
class << self
|
8
|
+
attr_accessor :verbose, :options
|
9
|
+
|
10
|
+
# The application options
|
11
|
+
def options(opts={})
|
12
|
+
@options ||= make_options(opts)
|
13
|
+
end
|
14
|
+
# Make the options with the config_file overrides included
|
15
|
+
# Default config file assumed to be at config/config.yml
|
16
|
+
def make_options(opts={})
|
17
|
+
loading_options = opts.delete(:optsparse) || {}
|
18
|
+
loading_options.merge!( {:argv => opts.delete(:argv)} )
|
19
|
+
|
20
|
+
load_options!(loading_options)
|
21
|
+
# default_options.merge!(opts)
|
22
|
+
# If the config_file options are specified and not empty
|
23
|
+
unless default_options[:config_file].nil? || default_options[:config_file].empty?
|
24
|
+
require "yaml"
|
25
|
+
# Try loading the file if it exists
|
26
|
+
filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
|
27
|
+
default_options.merge!( YAML.load(filedata) ) if filedata # We want the command-line to overwrite the config file
|
28
|
+
end
|
29
|
+
|
30
|
+
OpenStruct.new(default_options.merge(opts))
|
31
|
+
end
|
32
|
+
|
33
|
+
# Load options via commandline
|
34
|
+
def load_options!(opts={})
|
35
|
+
require 'optparse'
|
36
|
+
OptionParser.new do |op|
|
37
|
+
op.banner = opts[:banner] if opts[:banner]
|
38
|
+
op.on('-A key', '--access-key key', "Ec2 access key (ENV['ACCESS_KEY'])") { |key| default_options[:access_key] = key }
|
39
|
+
op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['SECRET_ACCESS_KEY'])") { |key| default_options[:secret_access_key] = key }
|
40
|
+
op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-4a46a323')") {|id| default_options[:ami] = id }
|
41
|
+
op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
|
42
|
+
op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }
|
43
|
+
op.on('-D ec2 directory', '--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
|
44
|
+
op.on('-r names', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
|
45
|
+
op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
|
46
|
+
op.on('-l plugin_dir', '--plugin-dir dir', "Plugin directory (default: '')") {|file| default_options[:plugin_dir] = file }
|
47
|
+
op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
|
48
|
+
op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }
|
49
|
+
op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
|
50
|
+
op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }
|
51
|
+
op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
|
52
|
+
op.on('-a address', '--public-ip address', "Associate this public address with the master node") {|s| default_options[:public_ip] = s}
|
53
|
+
op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
|
54
|
+
op.on('-n name', '--name name', "Application name") {|n| default_options[:app_name] = n}
|
55
|
+
op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
|
56
|
+
op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
|
57
|
+
op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
|
58
|
+
op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = true}
|
59
|
+
op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
|
60
|
+
op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x.to_i}
|
61
|
+
|
62
|
+
op.on_tail("-V", "Show version") do
|
63
|
+
puts Application.version
|
64
|
+
exit
|
65
|
+
end
|
66
|
+
op.on_tail("-h", "-?", "Show this message") do
|
67
|
+
puts op
|
68
|
+
exit
|
69
|
+
end
|
70
|
+
end.parse!(opts[:argv] ? opts.delete(:argv) : ARGV.dup)
|
71
|
+
end
|
72
|
+
|
73
|
+
# Basic default options
|
74
|
+
# All can be overridden by the command line
|
75
|
+
# or in a config.yml file
|
76
|
+
def default_options
|
77
|
+
@default_options ||= {
|
78
|
+
:app_name => "application_name",
|
79
|
+
:host_port => 80,
|
80
|
+
:client_port => 8001,
|
81
|
+
:environment => 'development',
|
82
|
+
:verbose => false,
|
83
|
+
:logging => true,
|
84
|
+
:size => "m1.small",
|
85
|
+
:polling_time => "30.seconds",
|
86
|
+
:user_data => "",
|
87
|
+
:heavy_load => 0.80,
|
88
|
+
:light_load => 0.15,
|
89
|
+
:minimum_instances => 2,
|
90
|
+
:maximum_instances => 4,
|
91
|
+
:public_ip => "",
|
92
|
+
:access_key => ENV["AWS_ACCESS_KEY_ID"],
|
93
|
+
:secret_access_key => ENV["AWS_SECRET_ACCESS_ID"],
|
94
|
+
:config_file => if ENV["CONFIG_FILE"] && !ENV["CONFIG_FILE"].empty?
|
95
|
+
ENV["CONFIG_FILE"]
|
96
|
+
elsif File.file?("config/config.yml")
|
97
|
+
"config/config.yml"
|
98
|
+
else
|
99
|
+
nil
|
100
|
+
end,
|
101
|
+
:username => "root",
|
102
|
+
:ec2_dir => ENV["EC2_HOME"],
|
103
|
+
:keypair => ENV["KEYPAIR_NAME"],
|
104
|
+
:ami => 'ami-4a46a323',
|
105
|
+
:shared_bucket => "",
|
106
|
+
:services => "",
|
107
|
+
:expand_when => "web_usage < 1.5\n memory > 0.85",
|
108
|
+
:contract_when => "cpu < 0.20\n memory < 0.10",
|
109
|
+
:os => "ubuntu",
|
110
|
+
:plugin_dir => "vendor",
|
111
|
+
:install_on_load => true
|
112
|
+
}
|
113
|
+
end
|
114
|
+
# Services monitored by Heartbeat
|
115
|
+
# Always at least monitors haproxy
|
116
|
+
def managed_services
|
117
|
+
"#{services}"
|
118
|
+
end
|
119
|
+
def master_managed_services
|
120
|
+
"cloud_master_takeover #{services}"
|
121
|
+
end
|
122
|
+
def launching_user_data
|
123
|
+
{:polling_time => polling_time}.to_yaml
|
124
|
+
end
|
125
|
+
# Keypair path
|
126
|
+
# Idiom:
|
127
|
+
# /Users/username/.ec2/id_rsa-name
|
128
|
+
def keypair_path
|
129
|
+
"#{ec2_dir}/id_rsa#{keypair ? "-#{keypair}" : "" }"
|
130
|
+
end
|
131
|
+
# Are we in development or test mode
|
132
|
+
%w(development production test).each do |env|
|
133
|
+
eval <<-EOE
|
134
|
+
def #{env}?
|
135
|
+
environment == '#{env}'
|
136
|
+
end
|
137
|
+
EOE
|
138
|
+
end
|
139
|
+
def environment=(env)
|
140
|
+
environment = env
|
141
|
+
end
|
142
|
+
def maintain_pid_path
|
143
|
+
"/var/run/pool_maintain.pid"
|
144
|
+
end
|
145
|
+
%w(scp_instances_script reconfigure_instances_script).each do |file|
|
146
|
+
define_method "sh_#{file}" do
|
147
|
+
File.join(File.dirname(__FILE__), "../..", "config", "#{file}.sh")
|
148
|
+
end
|
149
|
+
end
|
150
|
+
# Standard configuration files
|
151
|
+
%w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
|
152
|
+
define_method "#{file}_config_file" do
|
153
|
+
File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
|
154
|
+
end
|
155
|
+
end
|
156
|
+
def version
|
157
|
+
PoolParty::Version::STRING
|
158
|
+
end
|
159
|
+
def install_on_load?
|
160
|
+
options.install_on_load == true
|
161
|
+
end
|
162
|
+
# Call the options from the Application
|
163
|
+
def method_missing(m,*args)
|
164
|
+
options.methods.include?("#{m}") ? options.send(m,args) : super
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
end
|
169
|
+
|
170
|
+
end
|
@@ -0,0 +1,329 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
The basic master for PoolParty
|
3
|
+
=end
|
4
|
+
module PoolParty
|
5
|
+
class Master < Remoting
|
6
|
+
include Aska
|
7
|
+
include Callbacks
|
8
|
+
include Monitors
|
9
|
+
include Remoter
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
|
14
|
+
self.class.send :rules, :contract_when, Application.options.contract_when
|
15
|
+
self.class.send :rules, :expand_when, Application.options.expand_when
|
16
|
+
end
|
17
|
+
# Start the cloud
|
18
|
+
def start_cloud!
|
19
|
+
start!
|
20
|
+
end
|
21
|
+
alias_method :start_cloud, :start_cloud!
|
22
|
+
# Start the cloud, which launches the minimum_instances
|
23
|
+
def start!
|
24
|
+
message "Launching minimum_instances"
|
25
|
+
launch_minimum_instances
|
26
|
+
message "Waiting for master to boot up"
|
27
|
+
reset!
|
28
|
+
while !number_of_pending_instances.zero?
|
29
|
+
wait "2.seconds" unless Application.test?
|
30
|
+
waited = true
|
31
|
+
reset!
|
32
|
+
end
|
33
|
+
unless Application.test? || waited.nil?
|
34
|
+
message "Give some time for the instance ssh to start up"
|
35
|
+
wait "15.seconds"
|
36
|
+
end
|
37
|
+
install_cloud if Application.install_on_load?
|
38
|
+
configure_cloud
|
39
|
+
end
|
40
|
+
alias_method :start, :start!
|
41
|
+
def configure_cloud
|
42
|
+
message "Configuring master"
|
43
|
+
master = get_node 0
|
44
|
+
master.configure
|
45
|
+
end
|
46
|
+
def install_cloud
|
47
|
+
Master.with_nodes do |node|
|
48
|
+
node.login_once
|
49
|
+
end
|
50
|
+
Provider.install_poolparty(nodes.collect {|a| a.ip })
|
51
|
+
end
|
52
|
+
# Launch the minimum number of instances.
|
53
|
+
def launch_minimum_instances
|
54
|
+
request_launch_new_instances(Application.minimum_instances - number_of_pending_and_running_instances)
|
55
|
+
nodes
|
56
|
+
end
|
57
|
+
# Start monitoring the cloud with the threaded loop
|
58
|
+
def start_monitor!
|
59
|
+
begin
|
60
|
+
trap("INT") do
|
61
|
+
on_exit
|
62
|
+
exit
|
63
|
+
end
|
64
|
+
# Daemonize only if we are not in the test environment
|
65
|
+
run_thread_loop(:daemonize => !Application.test?) do
|
66
|
+
add_task {launch_minimum_instances} # If the base instances go down...
|
67
|
+
add_task {reconfigure_cloud_when_necessary}
|
68
|
+
add_task {scale_cloud!}
|
69
|
+
add_task {check_stats}
|
70
|
+
end
|
71
|
+
rescue Exception => e
|
72
|
+
puts "There was an error: #{e.nice_message}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
alias_method :start_monitor, :start_monitor!
|
76
|
+
def user_tasks
|
77
|
+
puts "in user_tasks"
|
78
|
+
end
|
79
|
+
# Sole purpose to check the stats, mainly in a plugin
|
80
|
+
def check_stats
|
81
|
+
end
|
82
|
+
# Add an instance if the cloud needs one ore terminate one if necessary
|
83
|
+
def scale_cloud!
|
84
|
+
add_instance_if_load_is_high
|
85
|
+
terminate_instance_if_load_is_low
|
86
|
+
end
|
87
|
+
alias_method :scale_cloud, :scale_cloud!
|
88
|
+
# Tough method:
|
89
|
+
# We need to make sure that all the instances have the required software installed
|
90
|
+
# This is a basic check against the local store of the instances that have the
|
91
|
+
# stack installed.
|
92
|
+
def reconfigure_cloud_when_necessary
|
93
|
+
reconfigure_running_instances if number_of_unconfigured_nodes > 0
|
94
|
+
end
|
95
|
+
alias_method :reconfiguration, :reconfigure_cloud_when_necessary
|
96
|
+
def number_of_unconfigured_nodes
|
97
|
+
nodes.reject {|a| a.stack_installed? }.size
|
98
|
+
end
|
99
|
+
def grow_by_one
|
100
|
+
request_launch_new_instance
|
101
|
+
self.class.get_master.configure
|
102
|
+
end
|
103
|
+
def shrink_by_one
|
104
|
+
node = nodes.reject {|a| a.master? }[-1]
|
105
|
+
request_termination_of_instance(node.instance_id) if node
|
106
|
+
end
|
107
|
+
# Add an instance if the load is high
|
108
|
+
def add_instance_if_load_is_high
|
109
|
+
request_launch_new_instance if expand?
|
110
|
+
end
|
111
|
+
alias_method :add_instance, :add_instance_if_load_is_high
|
112
|
+
# Teardown an instance if the load is pretty low
|
113
|
+
def terminate_instance_if_load_is_low
|
114
|
+
if contract?
|
115
|
+
shrink_by_one
|
116
|
+
end
|
117
|
+
end
|
118
|
+
alias_method :terminate_instance, :terminate_instance_if_load_is_low
|
119
|
+
# FOR MONITORING
|
120
|
+
def contract?
|
121
|
+
valid_rules?(:contract_when)
|
122
|
+
end
|
123
|
+
def expand?
|
124
|
+
valid_rules?(:expand_when)
|
125
|
+
end
|
126
|
+
# Restart the running instances services with monit on all the nodes
|
127
|
+
def restart_running_instances_services
|
128
|
+
nodes.each do |node|
|
129
|
+
node.restart_with_monit
|
130
|
+
end
|
131
|
+
end
|
132
|
+
# Reconfigure the running instances
|
133
|
+
# Since we are using vlad, running configure on one of the instances
|
134
|
+
# should configure all of the instances. We set the hosts in this file
|
135
|
+
def reconfigure_running_instances
|
136
|
+
# nodes.each do |node|
|
137
|
+
# node.configure if node.status =~ /running/
|
138
|
+
# end
|
139
|
+
master = get_node(0)
|
140
|
+
master.configure
|
141
|
+
end
|
142
|
+
# Build the basic haproxy config file from the config file in the config directory and return a tempfile
|
143
|
+
def build_haproxy_file
|
144
|
+
servers=<<-EOS
|
145
|
+
#{nodes.collect {|node| node.haproxy_entry}.join("\n")}
|
146
|
+
EOS
|
147
|
+
open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
|
148
|
+
end
|
149
|
+
# Build the hosts file and return a tempfile
|
150
|
+
def build_hosts_file
|
151
|
+
write_to_temp_file(nodes.collect {|a| a.hosts_entry }.join("\n"))
|
152
|
+
end
|
153
|
+
# Build host file for a specific node
|
154
|
+
def build_hosts_file_for(n)
|
155
|
+
servers=<<-EOS
|
156
|
+
#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}
|
157
|
+
EOS
|
158
|
+
servers
|
159
|
+
end
|
160
|
+
# Build the basic auth file for the heartbeat
|
161
|
+
def build_heartbeat_authkeys_file
|
162
|
+
write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read)
|
163
|
+
end
|
164
|
+
# Build heartbeat config file
|
165
|
+
def build_heartbeat_config_file_for(node)
|
166
|
+
servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}"
|
167
|
+
open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers}
|
168
|
+
end
|
169
|
+
# Return a list of the nodes and cache them
|
170
|
+
def nodes
|
171
|
+
@nodes ||= list_of_nonterminated_instances.collect_with_index do |inst, i|
|
172
|
+
RemoteInstance.new(inst.merge({:number => i}))
|
173
|
+
end
|
174
|
+
end
|
175
|
+
# Get the node at the specific index from the cached nodes
|
176
|
+
def get_node(i=0)
|
177
|
+
nodes.select {|a| a.number == i.to_i}.first
|
178
|
+
end
|
179
|
+
# Get the next node in sequence, so we can configure heartbeat to monitor the next node
|
180
|
+
def get_next_node(node)
|
181
|
+
i = node.number + 1
|
182
|
+
i = 0 if i >= (nodes.size - 1)
|
183
|
+
get_node(i)
|
184
|
+
end
|
185
|
+
# On exit command
|
186
|
+
def on_exit
|
187
|
+
end
|
188
|
+
# List the clouds
|
189
|
+
def list
|
190
|
+
if number_of_pending_and_running_instances > 0
|
191
|
+
out = "-- CLOUD (#{number_of_pending_and_running_instances})--\n"
|
192
|
+
out << nodes.collect {|node| node.description }.join("\n")
|
193
|
+
else
|
194
|
+
out = "Cloud is not running"
|
195
|
+
end
|
196
|
+
out
|
197
|
+
end
|
198
|
+
# Reset and clear the caches
|
199
|
+
def reset!
|
200
|
+
@cached_descriptions = nil
|
201
|
+
@nodes = nil
|
202
|
+
end
|
203
|
+
|
204
|
+
class << self
|
205
|
+
include PoolParty
|
206
|
+
|
207
|
+
def with_nodes(&block)
|
208
|
+
new.nodes.each &block
|
209
|
+
end
|
210
|
+
|
211
|
+
def collect_nodes(&block)
|
212
|
+
new.nodes.collect &block
|
213
|
+
end
|
214
|
+
|
215
|
+
def requires_heartbeat?
|
216
|
+
new.nodes.size > 1
|
217
|
+
end
|
218
|
+
def is_master_responding?
|
219
|
+
`ping -c1 -t5 #{get_master.ip}`
|
220
|
+
end
|
221
|
+
def get_master
|
222
|
+
new.nodes[0]
|
223
|
+
end
|
224
|
+
def get_next_node(node)
|
225
|
+
new.get_next_node(node)
|
226
|
+
end
|
227
|
+
# Build a heartbeat_config_file from the config file in the config directory and return a tempfile
|
228
|
+
def build_heartbeat_config_file_for(node)
|
229
|
+
return nil unless node
|
230
|
+
new.build_heartbeat_config_file_for(node)
|
231
|
+
end
|
232
|
+
# Build a heartbeat resources file from the config directory and return a tempfile
|
233
|
+
def build_heartbeat_resources_file_for(node)
|
234
|
+
return nil unless node
|
235
|
+
"#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}"
|
236
|
+
end
|
237
|
+
# Build hosts files for a specific node
|
238
|
+
def build_hosts_file_for(node)
|
239
|
+
new.build_hosts_file_for(node)
|
240
|
+
end
|
241
|
+
# Build the scp script for the specific node
|
242
|
+
def build_scp_instances_script_for(node)
|
243
|
+
authkeys_file = write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read.strip)
|
244
|
+
if Master.requires_heartbeat?
|
245
|
+
ha_d_file = Master.build_heartbeat_config_file_for(node)
|
246
|
+
haresources_file = Master.build_heartbeat_resources_file_for(node)
|
247
|
+
end
|
248
|
+
haproxy_file = Master.build_haproxy_file
|
249
|
+
hosts_file = Master.build_hosts_file_for(node)
|
250
|
+
|
251
|
+
str = open(Application.sh_scp_instances_script).read.strip ^ {
|
252
|
+
:cloud_master_takeover => "#{node.scp_string("#{root_dir}/config/cloud_master_takeover", "/etc/ha.d/resource.d/", :dir => "/etc/ha.d/resource.d")}",
|
253
|
+
:config_file => "#{node.scp_string(Application.config_file, "~/.config")}",
|
254
|
+
:authkeys => "#{node.scp_string(authkeys_file.path, "/etc/ha.d/authkeys", :dir => "/etc/ha.d/")}",
|
255
|
+
:resources => "#{node.scp_string("#{root_dir}/config/resource.d/*", "/etc/ha.d/resource.d/", {:switches => "-r"})}",
|
256
|
+
:monitrc => "#{node.scp_string(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")}",
|
257
|
+
:monit_d => "#{node.scp_string("#{File.dirname(Application.monit_config_file)}/monit/*", "/etc/monit.d/", {:switches => "-r", :dir => "/etc/monit.d/"})}",
|
258
|
+
:haproxy => "#{node.scp_string(haproxy_file, "/etc/haproxy.cfg")}",
|
259
|
+
|
260
|
+
:ha_d => Master.requires_heartbeat? ? "#{node.scp_string(ha_d_file, "/etc/ha.d/ha.cf")}" : "",
|
261
|
+
:haresources => Master.requires_heartbeat? ? "#{node.scp_string(haresources_file, "/etc/ha.d/ha.cf")}" : "",
|
262
|
+
|
263
|
+
:hosts => "#{node.scp_string(hosts_file, "/etc/hosts")}"
|
264
|
+
}
|
265
|
+
write_to_temp_file(str)
|
266
|
+
end
|
267
|
+
# Build basic configuration script for the node
|
268
|
+
def build_reconfigure_instances_script_for(node)
|
269
|
+
str = open(Application.sh_reconfigure_instances_script).read.strip ^ {
|
270
|
+
:config_master => "#{node.update_plugin_string}",
|
271
|
+
:start_pool_maintain => "pool maintain -c ~/.config -l ~/plugins",
|
272
|
+
:set_hostname => "hostname -v #{node.name}",
|
273
|
+
:start_s3fs => "/usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data"
|
274
|
+
}
|
275
|
+
write_to_temp_file(str)
|
276
|
+
end
|
277
|
+
|
278
|
+
def set_hosts(c, remotetask=nil)
|
279
|
+
unless remotetask.nil?
|
280
|
+
rt = remotetask
|
281
|
+
end
|
282
|
+
|
283
|
+
ssh_location = `which ssh`.gsub(/\n/, '')
|
284
|
+
rsync_location = `which rsync`.gsub(/\n/, '')
|
285
|
+
rt.set :user, Application.username
|
286
|
+
# rt.set :domain, "#{Application.user}@#{ip}"
|
287
|
+
rt.set :application, Application.app_name
|
288
|
+
rt.set :ssh_flags, "-i #{Application.keypair_path} -o StrictHostKeyChecking=no"
|
289
|
+
rt.set :rsync_flags , ['-azP', '--delete', "-e '#{ssh_location} -l #{Application.username} -i #{Application.keypair_path} -o StrictHostKeyChecking=no'"]
|
290
|
+
|
291
|
+
master = get_master
|
292
|
+
rt.set :domain, "#{master.ip}" if master
|
293
|
+
Master.with_nodes { |node|
|
294
|
+
rt.host "#{Application.username}@#{node.ip}",:app if node.status =~ /running/
|
295
|
+
}
|
296
|
+
end
|
297
|
+
|
298
|
+
def ssh_configure_string_for(node)
|
299
|
+
cmd=<<-EOC
|
300
|
+
#{node.update_plugin_string(node)}
|
301
|
+
pool maintain -c ~/.config -l #{PoolParty.plugin_dir}
|
302
|
+
hostname -v #{node.name}
|
303
|
+
/usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
|
304
|
+
EOC
|
305
|
+
end
|
306
|
+
def build_haproxy_file
|
307
|
+
servers=<<-EOS
|
308
|
+
#{collect_nodes {|node| node.haproxy_entry}.join("\n")}
|
309
|
+
EOS
|
310
|
+
open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
|
311
|
+
end
|
312
|
+
# Write a temp file with the content str
|
313
|
+
def write_to_temp_file(str="")
|
314
|
+
tempfile = Tempfile.new("pool-party-#{rand(1000)}-#{rand(1000)}")
|
315
|
+
tempfile.print(str)
|
316
|
+
tempfile.flush
|
317
|
+
tempfile
|
318
|
+
end
|
319
|
+
def with_temp_file(str="", &block)
|
320
|
+
Tempfile.open "pool-party-#{rand(10000)}" do |fp|
|
321
|
+
fp.puts str
|
322
|
+
fp.flush
|
323
|
+
block.call(fp)
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
|
328
|
+
end
|
329
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
Basic monitor on the cpu stats
|
3
|
+
=end
|
4
|
+
module Cpu
|
5
|
+
module Master
|
6
|
+
def cpu
|
7
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i+=a.cpu } / nodes.size : 0.0
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
module Remote
|
12
|
+
def cpu
|
13
|
+
ssh("uptime").split(/\s+/)[-3].to_f rescue 0.0
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
PoolParty.register_monitor Cpu
|
@@ -0,0 +1,26 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
Basic monitor on the cpu stats
|
3
|
+
=end
|
4
|
+
module Memory
|
5
|
+
module Master
|
6
|
+
# Get the average memory usage over the cloud
|
7
|
+
def memory
|
8
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.memory } / nodes.size : 0.0
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
module Remote
|
13
|
+
def memory
|
14
|
+
str = ssh("free -m | grep -i mem")
|
15
|
+
total_memory = str.split[1].to_f
|
16
|
+
used_memory = str.split[2].to_f
|
17
|
+
|
18
|
+
used_memory / total_memory
|
19
|
+
rescue
|
20
|
+
0.0
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
PoolParty.register_monitor Memory
|
@@ -0,0 +1,23 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
Basic monitor on the cpu stats
|
3
|
+
=end
|
4
|
+
module Web
|
5
|
+
module Master
|
6
|
+
# Get the average web request capabilities over the cloud
|
7
|
+
def web
|
8
|
+
nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.web } / nodes.size : 0.0
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
module Remote
|
13
|
+
def web
|
14
|
+
str = ssh("httperf --server localhost --port #{Application.client_port} --num-conn 3 --timeout 5 | grep 'Request rate'")
|
15
|
+
str[/[.]* ([\d]*\.[\d]*) [.]*/, 0].chomp.to_f
|
16
|
+
rescue
|
17
|
+
0.0
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
PoolParty.register_monitor Web
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module PoolParty
|
2
|
+
class Optioner
|
3
|
+
# Parse the command line options for options without a switch
|
4
|
+
def self.parse(argv, safe=[])
|
5
|
+
args = []
|
6
|
+
argv.each_with_index do |arg,i|
|
7
|
+
unless arg.index("-")
|
8
|
+
args << arg
|
9
|
+
else
|
10
|
+
argv.delete_at(i+1) unless safe.include?(arg)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
args
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|