auser-poolparty-extensions 0.0.7 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +15 -0
- data/Rakefile +2 -2
- data/VERSION.yml +2 -2
- data/lib/extensions/bash_alias.rb +3 -0
- data/lib/extensions/convenience_helpers.rb +89 -0
- data/lib/extensions/development_gem_package.rb +3 -2
- data/lib/extensions/ganglia/ganglia.rb +219 -0
- data/lib/extensions/ganglia/templates/bin/gmetad.erb +38 -0
- data/lib/extensions/ganglia/templates/bin/gmond.erb +38 -0
- data/lib/extensions/ganglia/templates/ganglia-web-conf.php.erb +191 -0
- data/lib/extensions/ganglia/templates/gmetad.conf.erb +116 -0
- data/lib/extensions/ganglia/templates/gmond.conf.erb +426 -0
- data/lib/extensions/ganglia/templates/hadoop-metrics.properties.erb +65 -0
- data/lib/extensions/hadoop/hadoop.rb +427 -0
- data/lib/extensions/hadoop/templates/core-site.xml.erb +62 -0
- data/lib/extensions/hadoop/templates/hadoop-env.sh +4 -0
- data/lib/extensions/hadoop/templates/hadoop-site.xml.erb +9 -0
- data/lib/extensions/hadoop/templates/hadoop_hosts.erb +0 -0
- data/lib/extensions/hadoop/templates/hdfs-site.xml.erb +48 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-datanode +125 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-jobtracker +125 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-namenode +124 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-secondarynamenode +124 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-tasktracker +124 -0
- data/lib/extensions/hadoop/templates/jvm.conf +12 -0
- data/lib/extensions/hadoop/templates/log4j.properties.erb +94 -0
- data/lib/extensions/hadoop/templates/mapred-site.xml.erb +70 -0
- data/lib/extensions/hive/hive.rb +138 -0
- data/lib/poolparty-extensions.rb +5 -2
- metadata +28 -4
data/README.rdoc
CHANGED
@@ -10,6 +10,21 @@
|
|
10
10
|
|
11
11
|
= Available extensions
|
12
12
|
|
13
|
+
== bash_alias
|
14
|
+
Adds a bash alias
|
15
|
+
|
16
|
+
== development_gem_package
|
17
|
+
Deploy a and install a gem developed locally. This is useful when you are
|
18
|
+
developing an internal or forked gem and you want to deploy it to your cloud
|
19
|
+
|
20
|
+
|
21
|
+
== Usage
|
22
|
+
|
23
|
+
has_development_gem_package('jetty-rack',
|
24
|
+
:from => "~/path/to/my/site",
|
25
|
+
:git_pull_first => true) # git pull from before sending to server
|
26
|
+
|
27
|
+
|
13
28
|
== dynomite
|
14
29
|
Installs edge dynomite from the git repository
|
15
30
|
|
data/Rakefile
CHANGED
@@ -8,7 +8,7 @@ begin
|
|
8
8
|
gem.summary = %Q{Extensions on to of poolparty}
|
9
9
|
gem.email = "arilerner@mac.com"
|
10
10
|
gem.homepage = "http://github.com/auser/poolparty-extensions"
|
11
|
-
gem.authors = ["Ari Lerner"]
|
11
|
+
gem.authors = ["Ari Lerner, Nate Murray, & Michael Fairchild"]
|
12
12
|
|
13
13
|
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
14
14
|
end
|
@@ -76,7 +76,7 @@ task :readme do
|
|
76
76
|
footer =<<-EOF
|
77
77
|
== Copyright
|
78
78
|
|
79
|
-
Copyright (c) 2009 Ari Lerner. See LICENSE for details.
|
79
|
+
Copyright (c) 2009 Ari Lerner & Cloudteam. See LICENSE for details.
|
80
80
|
EOF
|
81
81
|
|
82
82
|
extensions = FileList["#{File.dirname(__FILE__)}/lib/extensions/*.rb"]
|
data/VERSION.yml
CHANGED
@@ -0,0 +1,89 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
|
3
|
+
== Convenience Helpers
|
4
|
+
|
5
|
+
Adds a few handy packages and aliases for developing poolparty instances.
|
6
|
+
|
7
|
+
== Usage
|
8
|
+
|
9
|
+
has_convenience_helpers
|
10
|
+
|
11
|
+
== Examples
|
12
|
+
|
13
|
+
$ inspect-poolparty-recipes # => will vim the poolparty chef recipe file
|
14
|
+
$ cd-cookbooks # => cd into the poolparty cookbooks directory
|
15
|
+
$ tree # => show the directory structure as a nice tree
|
16
|
+
/var/poolparty/dr_configure/chef/cookbooks/poolparty# tree
|
17
|
+
.
|
18
|
+
|-- attributes
|
19
|
+
| `-- poolparty.rb
|
20
|
+
|-- recipes
|
21
|
+
| `-- default.rb
|
22
|
+
`-- templates
|
23
|
+
`-- default
|
24
|
+
|-- etc
|
25
|
+
| |-- jvm.erb
|
26
|
+
| `-- motd.erb
|
27
|
+
|-- home
|
28
|
+
| `-- hadoop
|
29
|
+
| `-- ssh
|
30
|
+
| `-- hadoop_id_rsa.erb
|
31
|
+
`-- usr
|
32
|
+
`-- local
|
33
|
+
`-- hadoop
|
34
|
+
`-- conf
|
35
|
+
|-- hadoop-env.sh.erb
|
36
|
+
`-- hadoop-site.xml.erb
|
37
|
+
|
38
|
+
=end
|
39
|
+
|
40
|
+
module PoolParty
|
41
|
+
module Plugin
|
42
|
+
class ConvenienceHelpers < Plugin
|
43
|
+
def before_load(o={}, &block)
|
44
|
+
add_packages
|
45
|
+
add_aliases
|
46
|
+
add_binaries
|
47
|
+
add_profile_updates
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_packages
|
51
|
+
has_package "curl"
|
52
|
+
has_package "tree"
|
53
|
+
has_package "screen"
|
54
|
+
|
55
|
+
case cloud.os
|
56
|
+
when "centos"
|
57
|
+
has_package "ruby-irb"
|
58
|
+
else
|
59
|
+
has_package "irb"
|
60
|
+
has_package "vim-nox"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def add_aliases
|
65
|
+
has_bash_alias :name => "inspect-poolparty-recipes", :value => "vi /var/poolparty/dr_configure/chef/cookbooks/poolparty/recipes/default.rb"
|
66
|
+
has_bash_alias :name => "cd-cookbooks", :value => "pushd /var/poolparty/dr_configure/chef/cookbooks/poolparty"
|
67
|
+
|
68
|
+
%w{instance-id local-hostname local-ipv4 public-hostname public-ipv4 security-groups}.each do |metaalias|
|
69
|
+
has_bash_alias :name => metaalias, :value => "curl http://169.254.169.254/latest/meta-data/#{metaalias}"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def add_binaries
|
74
|
+
has_exec "wget http://gist.github.com/raw/131294/0622454b2cc2f787c04d20ab3d47e888e31edcd4/gistfile1 -O /usr/bin/xtail && chmod +x /usr/bin/xtail",
|
75
|
+
:not_if => "test -e /usr/bin/xtail"
|
76
|
+
has_exec "curl http://timkay.com/aws/aws -o /usr/bin/aws", :not_if => "test -e /usr/bin/aws"
|
77
|
+
end
|
78
|
+
|
79
|
+
def add_profile_updates
|
80
|
+
has_exec %Q{echo \\"export PS1='\\\\u@\\\\h \\\\A \\\\w (#{cloud_name}) $ '\\" >> /root/.profile}, :not_if => "grep PS1 /root/.profile | grep #{cloud_name}"
|
81
|
+
# bind '"\e[A":history-search-backward'
|
82
|
+
# bind '"\e[B":history-search-forward'
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
|
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
=begin rdoc
|
1
|
+
=begin rdoc
|
3
2
|
== Development Gem
|
4
3
|
|
5
4
|
Deploy a and install a gem developed locally. This is useful when you are
|
@@ -13,6 +12,8 @@ developing an internal or forked gem and you want to deploy it to your cloud
|
|
13
12
|
:git_pull_first => true) # git pull from before sending to server
|
14
13
|
|
15
14
|
=end
|
15
|
+
|
16
|
+
module PoolParty
|
16
17
|
module Plugin
|
17
18
|
class DevelopmentGem < Plugin
|
18
19
|
|
@@ -0,0 +1,219 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
|
3
|
+
== Overview
|
4
|
+
Install Ganglia cloud monitoring system
|
5
|
+
|
6
|
+
== Requirements
|
7
|
+
You'll need apache and php enabled in your clouds.rb. For example:
|
8
|
+
|
9
|
+
apache do
|
10
|
+
enable_php5 do
|
11
|
+
extras :gd
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
Because the configs need to know about every node in the cloud *after* it has
|
16
|
+
launched, you must setup an after_all_loaded block in your clouds.rb that calls
|
17
|
+
ganglia.perform_after_all_loaded_for_master. For example:
|
18
|
+
|
19
|
+
after_all_loaded do
|
20
|
+
clouds[:hadoop_master].run_in_context do
|
21
|
+
ganglia.perform_after_all_loaded_for_master
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
Currently the tasks only need to be run for master, so simply call this on your
|
26
|
+
"master" cloud. Note: replace hadoop_master with the name of your cloud above.
|
27
|
+
|
28
|
+
== EC2 Firewall
|
29
|
+
|
30
|
+
ec2-authorize -P udp -p 8649 nmurray-hadoop
|
31
|
+
ec2-authorize -P tcp -p 8649 nmurray-hadoop
|
32
|
+
|
33
|
+
== References
|
34
|
+
* http://www.ibm.com/developerworks/wikis/display/WikiPtype/ganglia?decorator=printable
|
35
|
+
* http://docs.google.com/Doc?id=dgmmft5s_45hr7hmggr
|
36
|
+
* http://www.hps.com/~tpg/notebook/ganglia.php
|
37
|
+
* http://www.cultofgary.com/2008/10/16/ec2-and-ganglia/
|
38
|
+
=end
|
39
|
+
|
40
|
+
module PoolParty
|
41
|
+
module Plugin
|
42
|
+
class Ganglia < Plugin
|
43
|
+
def before_load(o={}, &block)
|
44
|
+
@monitored_features ||= {}
|
45
|
+
do_once do
|
46
|
+
install_dependencies
|
47
|
+
download
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def install_dependencies
|
52
|
+
has_package :name => "rrdtool"
|
53
|
+
has_package :name => "build-essential"
|
54
|
+
has_package :name => "librrd-dev"
|
55
|
+
has_package :name => "libapr1-dev"
|
56
|
+
has_package :name => "libconfuse-dev"
|
57
|
+
has_package :name => "libexpat1-dev"
|
58
|
+
has_package :name => "python-dev"
|
59
|
+
|
60
|
+
has_group "ganglia", :action => :create
|
61
|
+
has_user "ganglia", :gid => "ganglia"
|
62
|
+
|
63
|
+
# libart-2.0-2 ?
|
64
|
+
end
|
65
|
+
|
66
|
+
def download
|
67
|
+
has_exec "wget http://superb-west.dl.sourceforge.net/sourceforge/ganglia/ganglia-3.1.2.tar.gz -O /usr/local/src/ganglia-3.1.2.tar.gz",
|
68
|
+
:not_if => "test -e /usr/local/src/ganglia-3.1.2.tar.gz"
|
69
|
+
has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/ganglia-3.1.2.tar.gz",
|
70
|
+
:not_if => "test -e /usr/local/src/ganglia-3.1.2"
|
71
|
+
end
|
72
|
+
|
73
|
+
def master
|
74
|
+
has_exec "cd /usr/local/src/ganglia-3.1.2 && ./configure --with-gmetad && make && make install",
|
75
|
+
:not_if => "test -e /usr/lib/ganglia"
|
76
|
+
has_exec "mv /usr/local/src/ganglia-3.1.2/web /var/www/ganglia",
|
77
|
+
:not_if => "test -e /var/www/ganglia"
|
78
|
+
has_file :name => "/var/www/ganglia/conf.php", :mode => "0644", :template => "ganglia-web-conf.php.erb"
|
79
|
+
has_variable "ganglia_gmond_is_master", :value => true
|
80
|
+
gmond
|
81
|
+
gmetad
|
82
|
+
end
|
83
|
+
|
84
|
+
def slave
|
85
|
+
has_exec "cd /usr/local/src/ganglia-3.1.2 && ./configure && make && make install",
|
86
|
+
:not_if => "test -e /usr/lib/ganglia"
|
87
|
+
has_variable "ganglia_gmond_is_master", :value => false
|
88
|
+
gmond
|
89
|
+
end
|
90
|
+
|
91
|
+
def gmond
|
92
|
+
has_directory "/etc/ganglia"
|
93
|
+
has_exec({:name => "restart-gmond", :command => "/etc/init.d/gmond restart", :action => :nothing})
|
94
|
+
|
95
|
+
has_file(:name => "/etc/init.d/gmond") do
|
96
|
+
mode 0755
|
97
|
+
template :bin/"gmond.erb"
|
98
|
+
notifies get_exec("restart-gmond"), :run
|
99
|
+
end
|
100
|
+
|
101
|
+
install_extra_gmond_monitors
|
102
|
+
end
|
103
|
+
|
104
|
+
def gmetad
|
105
|
+
has_directory "/var/lib/ganglia/rrds"
|
106
|
+
has_exec "chmod 755 /var/lib/ganglia/rrds"
|
107
|
+
has_exec "chown -R ganglia:ganglia /var/lib/ganglia/rrds"
|
108
|
+
has_exec({:name => "restart-gmetad", :command => "/etc/init.d/gmetad restart", :action => :nothing})
|
109
|
+
has_file(:name => "/etc/init.d/gmetad") do
|
110
|
+
mode 0755
|
111
|
+
template :bin/"gmetad.erb"
|
112
|
+
notifies get_exec("restart-gmetad"), :run
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def monitor(*cloud_names)
|
117
|
+
@monitored_clouds = cloud_names
|
118
|
+
end
|
119
|
+
|
120
|
+
def perform_after_all_loaded_for_slave
|
121
|
+
gmond_after_all_loaded
|
122
|
+
end
|
123
|
+
|
124
|
+
def perform_after_all_loaded_for_master
|
125
|
+
raise "No clouds to monitor with ganglia specified. Please use the 'monitor(*cloud_names)' directive within your ganglia block" unless @monitored_clouds
|
126
|
+
gmond_after_all_loaded
|
127
|
+
|
128
|
+
data_sources = ""
|
129
|
+
@monitored_clouds.each do |cloud_name|
|
130
|
+
line = "data_source \\\"#{cloud_name}\\\" "
|
131
|
+
ips = []
|
132
|
+
if clouds[cloud_name.intern]
|
133
|
+
clouds[cloud_name.intern].nodes(:status => 'running').each_with_index do |n, i|
|
134
|
+
ips << (n[:private_dns_name] || n[:ip]) + ":8649" # todo - what if we used master0, slave0 etc here?
|
135
|
+
end
|
136
|
+
end
|
137
|
+
data_sources << (line + ips.join(" ") + "\n")
|
138
|
+
end
|
139
|
+
data_sources.gsub!(/\n/, '\n')
|
140
|
+
|
141
|
+
has_variable "ganglia_gmetad_data_sources", :value => data_sources
|
142
|
+
|
143
|
+
# poolname = clouds.values.first.pool.name
|
144
|
+
# has_variable "ganglia_pool_name", :value => (poolname && !poolname.empty? ? poolname : "pool")
|
145
|
+
has_variable "ganglia_pool_name", :value => "pool"
|
146
|
+
|
147
|
+
has_exec(:name => "restart_gmetad2", :command => "/etc/init.d/gmetad restart", :action => :nothing) # HACK this is already defined!, todo
|
148
|
+
has_file(:name => "/etc/ganglia/gmetad.conf", :mode => 0644, :template => "gmetad.conf.erb" do
|
149
|
+
notifies get_exec("restart_gmetad2"), :run
|
150
|
+
end
|
151
|
+
has_service "gmetad", :enabled => true, :running => true, :supports => [:restart]
|
152
|
+
end
|
153
|
+
|
154
|
+
def track(*features)
|
155
|
+
@monitored_features ||= {}
|
156
|
+
features.map {|f| @monitored_features[f] = true }
|
157
|
+
end
|
158
|
+
|
159
|
+
def gmond_after_all_loaded
|
160
|
+
has_variable "ganglia_cloud_name", :value => cloud_name
|
161
|
+
has_variable "ganglia_this_nodes_private_ip", :value => lambda{ %Q{%x[curl http://169.254.169.254/latest/meta-data/local-ipv4]}}
|
162
|
+
has_variable "ganglia_masters_ip", :value => lambda { %Q{\`ping -c1 master0 | grep PING | awk -F '[()]' '{print $2 }'\`.strip}}
|
163
|
+
|
164
|
+
first_node = clouds[cloud_name].nodes(:status => 'running').first
|
165
|
+
|
166
|
+
|
167
|
+
if first_node
|
168
|
+
has_variable "ganglia_first_node_in_clusters_ip", :value => lambda { %Q{\`ping -c1 #{first_node[:private_dns_name]} | grep PING | awk -F '[()]' '{print $2 }'\`.strip}}
|
169
|
+
|
170
|
+
has_file(:name => "/etc/ganglia/gmond.conf") do
|
171
|
+
mode 0644
|
172
|
+
template "gmond.conf.erb"
|
173
|
+
# notifies get_exec("restart-gmond"), :run
|
174
|
+
end
|
175
|
+
|
176
|
+
enable_tracking_configs
|
177
|
+
|
178
|
+
end
|
179
|
+
has_service "gmond", :enabled => true, :running => true, :supports => [:restart]
|
180
|
+
|
181
|
+
end
|
182
|
+
|
183
|
+
def enable_tracking_configs
|
184
|
+
if @monitored_features[:hadoop]
|
185
|
+
has_variable "hadoop_ganglia_monitoring_enabled", :value => true
|
186
|
+
|
187
|
+
# hmm, should maybe be mvd to hadoop plugin?
|
188
|
+
has_file(:name => "/usr/local/hadoop/conf/hadoop-metrics.properties") do
|
189
|
+
mode 0644
|
190
|
+
template "hadoop-metrics.properties.erb"
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
def install_extra_gmond_monitors
|
196
|
+
has_directory(:name => "/etc/ganglia/bin/monitors")
|
197
|
+
%w{sshd_ganglia}.each do |monitor|
|
198
|
+
has_file(:name => "/etc/ganglia/bin/monitors/#{monitor}") do
|
199
|
+
mode 0755
|
200
|
+
template "bin/monitors/#{monitor}.sh"
|
201
|
+
end
|
202
|
+
has_cron( :name => "ganglia_monitor_#{monitor}",
|
203
|
+
:command => "bash -c /etc/ganglia/bin/monitors/#{monitor}",
|
204
|
+
:user => "root",
|
205
|
+
:minute => "*/5",
|
206
|
+
:hour => "*",
|
207
|
+
:month => "*",
|
208
|
+
:weekday => "*")
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
# todo, add a verifier
|
213
|
+
# telnet localhost 8649
|
214
|
+
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
4
|
+
DAEMON=/usr/sbin/gmetad
|
5
|
+
NAME=gmetad
|
6
|
+
DESC="Ganglia Monitor Meta-Daemon"
|
7
|
+
|
8
|
+
test -x $DAEMON || exit 0
|
9
|
+
set -e
|
10
|
+
|
11
|
+
case "$1" in
|
12
|
+
start)
|
13
|
+
echo -n "Starting $DESC: "
|
14
|
+
start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
|
15
|
+
--exec $DAEMON
|
16
|
+
echo "$NAME."
|
17
|
+
;;
|
18
|
+
stop)
|
19
|
+
echo -n "Stopping $DESC: "
|
20
|
+
start-stop-daemon --stop --quiet --oknodo \
|
21
|
+
--exec $DAEMON 2>&1 > /dev/null
|
22
|
+
echo "$NAME."
|
23
|
+
;;
|
24
|
+
reload)
|
25
|
+
;;
|
26
|
+
restart|force-reload)
|
27
|
+
$0 stop
|
28
|
+
$0 start
|
29
|
+
;;
|
30
|
+
*)
|
31
|
+
N=/etc/init.d/$NAME
|
32
|
+
# echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
|
33
|
+
echo "Usage: $N {start|stop|restart|force-reload}" >&2
|
34
|
+
exit 1
|
35
|
+
;;
|
36
|
+
esac
|
37
|
+
|
38
|
+
exit 0
|
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
4
|
+
DAEMON=/usr/sbin/gmond
|
5
|
+
NAME=gmond
|
6
|
+
DESC="Ganglia Monitor Daemon"
|
7
|
+
|
8
|
+
test -x $DAEMON || exit 0
|
9
|
+
set -e
|
10
|
+
|
11
|
+
case "$1" in
|
12
|
+
start)
|
13
|
+
echo -n "Starting $DESC: "
|
14
|
+
start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
|
15
|
+
--exec $DAEMON
|
16
|
+
echo "$NAME."
|
17
|
+
;;
|
18
|
+
stop)
|
19
|
+
echo -n "Stopping $DESC: "
|
20
|
+
start-stop-daemon --stop --quiet --oknodo \
|
21
|
+
--exec $DAEMON 2>&1 > /dev/null
|
22
|
+
echo "$NAME."
|
23
|
+
;;
|
24
|
+
reload)
|
25
|
+
;;
|
26
|
+
restart|force-reload)
|
27
|
+
$0 stop
|
28
|
+
$0 start
|
29
|
+
;;
|
30
|
+
*)
|
31
|
+
N=/etc/init.d/$NAME
|
32
|
+
# echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
|
33
|
+
echo "Usage: $N {start|stop|restart|force-reload}" >&2
|
34
|
+
exit 1
|
35
|
+
;;
|
36
|
+
esac
|
37
|
+
|
38
|
+
exit 0
|