jashmenn-poolparty-extensions 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION.yml +1 -1
- data/lib/extensions/convenience_helpers.rb +4 -0
- data/lib/extensions/ganglia/ganglia.rb +22 -0
- data/lib/extensions/ganglia/templates/hadoop-metrics.properties.erb +65 -0
- data/lib/extensions/hadoop/hadoop.rb +319 -0
- data/lib/extensions/hadoop/templates/core-site.xml.erb +64 -0
- data/lib/extensions/hadoop/templates/hadoop-env.sh +3 -0
- data/lib/extensions/hadoop/templates/hadoop-site.xml.erb +9 -0
- data/lib/extensions/hadoop/templates/hadoop_hosts.erb +0 -0
- data/lib/extensions/hadoop/templates/hdfs-site.xml.erb +47 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-datanode +119 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-jobtracker +119 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-namenode +119 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-secondarynamenode +119 -0
- data/lib/extensions/hadoop/templates/init.d/hadoop-tasktracker +119 -0
- data/lib/extensions/hadoop/templates/jvm.conf +12 -0
- data/lib/extensions/hadoop/templates/log4j.properties.erb +94 -0
- data/lib/extensions/hadoop/templates/mapred-site.xml.erb +70 -0
- data/lib/extensions/hive/hive.rb +118 -0
- metadata +18 -2
data/VERSION.yml
CHANGED
@@ -57,6 +57,10 @@ module PoolParty
|
|
57
57
|
def add_aliases
|
58
58
|
has_bash_alias :name => "inspect-poolparty-recipes", :value => "vi /var/poolparty/dr_configure/chef/cookbooks/poolparty/recipes/default.rb"
|
59
59
|
has_bash_alias :name => "cd-cookbooks", :value => "pushd /var/poolparty/dr_configure/chef/cookbooks/poolparty"
|
60
|
+
|
61
|
+
%w{instance-id local-hostname local-ipv4 public-hostname public-ipv4 security-groups}.each do |metaalias|
|
62
|
+
has_bash_alias :name => metaalias, :value => "curl http://169.254.169.254/latest/meta-data/#{metaalias}"
|
63
|
+
end
|
60
64
|
end
|
61
65
|
|
62
66
|
def add_binaries
|
@@ -41,6 +41,7 @@ module PoolParty
|
|
41
41
|
module Plugin
|
42
42
|
class Ganglia < Plugin
|
43
43
|
def before_load(o={}, &block)
|
44
|
+
@monitored_features ||= {}
|
44
45
|
do_once do
|
45
46
|
install_dependencies
|
46
47
|
download
|
@@ -143,6 +144,11 @@ module PoolParty
|
|
143
144
|
has_service "gmetad", :enabled => true, :running => true, :supports => [:restart]
|
144
145
|
end
|
145
146
|
|
147
|
+
def track(*features)
|
148
|
+
@monitored_features ||= {}
|
149
|
+
features.map {|f| @monitored_features[f] = true }
|
150
|
+
end
|
151
|
+
|
146
152
|
def gmond_after_all_loaded
|
147
153
|
has_variable "ganglia_cloud_name", :value => cloud_name
|
148
154
|
has_variable "ganglia_this_nodes_private_ip", :value => lambda{ %Q{%x[curl http://169.254.169.254/latest/meta-data/local-ipv4]}}
|
@@ -150,6 +156,7 @@ module PoolParty
|
|
150
156
|
|
151
157
|
first_node = clouds[cloud_name].nodes(:status => 'running').first
|
152
158
|
|
159
|
+
|
153
160
|
if first_node
|
154
161
|
has_variable "ganglia_first_node_in_clusters_ip", :value => lambda { %Q{\`ping -c1 #{first_node[:private_dns_name]} | grep PING | awk -F '[()]' '{print $2 }'\`.strip}}
|
155
162
|
|
@@ -158,11 +165,26 @@ module PoolParty
|
|
158
165
|
template "gmond.conf.erb"
|
159
166
|
# calls get_exec("restart-gmond")
|
160
167
|
end
|
168
|
+
|
169
|
+
enable_tracking_configs
|
170
|
+
|
161
171
|
end
|
162
172
|
has_service "gmond", :enabled => true, :running => true, :supports => [:restart]
|
163
173
|
|
164
174
|
end
|
165
175
|
|
176
|
+
def enable_tracking_configs
|
177
|
+
if @monitored_features[:hadoop]
|
178
|
+
has_variable "hadoop_ganglia_monitoring_enabled", :value => true
|
179
|
+
|
180
|
+
# hmm, should maybe be mvd to hadoop plugin?
|
181
|
+
has_file(:name => "/usr/local/hadoop/conf/hadoop-metrics.properties") do
|
182
|
+
mode 0644
|
183
|
+
template "hadoop-metrics.properties.erb"
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
166
188
|
# todo, add a verifier
|
167
189
|
# telnet localhost 8649
|
168
190
|
|
@@ -0,0 +1,65 @@
|
|
1
|
+
|
2
|
+
# Configuration of the "dfs" context for null
|
3
|
+
dfs.class=org.apache.hadoop.metrics.spi.NullContext
|
4
|
+
|
5
|
+
# Configuration of the "dfs" context for file
|
6
|
+
#dfs.class=org.apache.hadoop.metrics.file.FileContext
|
7
|
+
#dfs.period=10
|
8
|
+
#dfs.fileName=/tmp/dfsmetrics.log
|
9
|
+
|
10
|
+
# Configuration of the "dfs" context for ganglia
|
11
|
+
# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
|
12
|
+
# dfs.period=10
|
13
|
+
# dfs.servers=localhost:8649
|
14
|
+
|
15
|
+
|
16
|
+
# Configuration of the "mapred" context for null
|
17
|
+
mapred.class=org.apache.hadoop.metrics.spi.NullContext
|
18
|
+
|
19
|
+
# Configuration of the "mapred" context for file
|
20
|
+
#mapred.class=org.apache.hadoop.metrics.file.FileContext
|
21
|
+
#mapred.period=10
|
22
|
+
#mapred.fileName=/tmp/mrmetrics.log
|
23
|
+
|
24
|
+
# Configuration of the "mapred" context for ganglia
|
25
|
+
# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
|
26
|
+
# mapred.period=10
|
27
|
+
# mapred.servers=localhost:8649
|
28
|
+
|
29
|
+
|
30
|
+
# Configuration of the "jvm" context for null
|
31
|
+
jvm.class=org.apache.hadoop.metrics.spi.NullContext
|
32
|
+
|
33
|
+
# Configuration of the "jvm" context for file
|
34
|
+
#jvm.class=org.apache.hadoop.metrics.file.FileContext
|
35
|
+
#jvm.period=10
|
36
|
+
#jvm.fileName=/tmp/jvmmetrics.log
|
37
|
+
|
38
|
+
# Configuration of the "jvm" context for ganglia
|
39
|
+
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
|
40
|
+
# jvm.period=10
|
41
|
+
# jvm.servers=localhost:8649
|
42
|
+
#
|
43
|
+
<% if @node[:poolparty][:hadoop_ganglia_monitoring_enabled] %>
|
44
|
+
|
45
|
+
# The value in the udp_send_channel you use in your /etc/gmond.conf (look for the line which says mcast_join=<IP ADDRESS>).
|
46
|
+
|
47
|
+
# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
|
48
|
+
# dfs.period=10
|
49
|
+
# dfs.servers=<%= @node[:poolparty][:ganglia_first_node_in_clusters_ip] %>:8649
|
50
|
+
|
51
|
+
# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
|
52
|
+
# mapred.period=10
|
53
|
+
# mapred.servers=<%= @node[:poolparty][:ganglia_first_node_in_clusters_ip] %>:8649
|
54
|
+
|
55
|
+
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
|
56
|
+
# jvm.period=10
|
57
|
+
# jvm.servers=<%= @node[:poolparty][:ganglia_first_node_in_clusters_ip] %>:8649
|
58
|
+
|
59
|
+
# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
|
60
|
+
# rpc.period=10
|
61
|
+
# rpc.servers=<%= @node[:poolparty][:ganglia_first_node_in_clusters_ip] %>:8649
|
62
|
+
|
63
|
+
<% end %>
|
64
|
+
|
65
|
+
# vim: ft=conf
|
@@ -0,0 +1,319 @@
|
|
1
|
+
=begin rdoc
|
2
|
+
|
3
|
+
== Overview
|
4
|
+
Install a hadoop cluster
|
5
|
+
|
6
|
+
== Requirements
|
7
|
+
You'll need apache and php enabled in your clouds.rb. For example:
|
8
|
+
|
9
|
+
apache do
|
10
|
+
enable_php5
|
11
|
+
end
|
12
|
+
|
13
|
+
== Bugs
|
14
|
+
This assumes your clouds are named "hadoop_master" and "hadoop_slave". That sucks. TODO: pass these in as variables
|
15
|
+
|
16
|
+
== References
|
17
|
+
=end
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
module PoolParty
|
22
|
+
module Plugin
|
23
|
+
class Hadoop < Plugin
|
24
|
+
def before_load(o={}, &block)
|
25
|
+
do_once do
|
26
|
+
install_jdk
|
27
|
+
# add_users_and_groups
|
28
|
+
create_keys
|
29
|
+
connect_keys
|
30
|
+
build
|
31
|
+
configure
|
32
|
+
format_hdfs
|
33
|
+
create_aliases
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def perform_just_in_time_operations
|
38
|
+
create_reference_hosts
|
39
|
+
create_ssh_configs
|
40
|
+
create_master_and_slaves_files
|
41
|
+
end
|
42
|
+
|
43
|
+
def install_jdk
|
44
|
+
# accept the sun license agreements. see: http://www.davidpashley.com/blog/debian/java-license
|
45
|
+
has_exec "echo sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true | /usr/bin/debconf-set-selections"
|
46
|
+
has_exec "echo sun-java6-jre shared/accepted-sun-dlj-v1-1 select true | /usr/bin/debconf-set-selections"
|
47
|
+
has_package(:name => "sun-java6-jdk")
|
48
|
+
has_file(:name => "/etc/jvm") do
|
49
|
+
mode 0644
|
50
|
+
template "jvm.conf"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def add_users_and_groups
|
55
|
+
has_group "hadoop", :action => :create
|
56
|
+
has_user "hadoop", :gid => "hadoop"
|
57
|
+
has_directory "/home/hadoop", :owner => "hadoop", :mode => "755"
|
58
|
+
|
59
|
+
# TODO - ssh key code below needs to turn into these lines. those should become plugins
|
60
|
+
# has_ssh_key :user => "hadoop", :name => "hadoop_id_rsa", :create => true
|
61
|
+
# has_authorized_key :user => "hadoop", :name => "hadoop_id_rsa"
|
62
|
+
end
|
63
|
+
|
64
|
+
def create_keys
|
65
|
+
unless File.exists?(hadoop_id_rsa)
|
66
|
+
FileUtils.mkdir_p(cloud_keys_dir)
|
67
|
+
cmd = "ssh-keygen -t rsa -N '' -f #{hadoop_id_rsa}" # todo, make into variables
|
68
|
+
puts cmd
|
69
|
+
`#{cmd}`
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# everything below should become methods and/or plugins
|
74
|
+
def connect_keys
|
75
|
+
# has_exec "ssh-keygen -t rsa -N '' -f /home/hadoop/.ssh/id_rsa", :user => "hadoop", :not_if => "test -e /home/hadoop/.ssh/id_rsa"
|
76
|
+
|
77
|
+
# so annoying, chef/rsync/something doesn't copy over dotfiles, so upload it as non-dot
|
78
|
+
has_directory :name => "#{home_dir}/ssh"
|
79
|
+
has_directory :name => "#{home_dir}/.ssh"
|
80
|
+
has_file :name => "#{home_dir}/ssh/#{hadoop_id_rsa_base}", :content => open(hadoop_id_rsa).read
|
81
|
+
has_exec "mv #{home_dir}/ssh/hadoop_id_rsa #{home_dir}/.ssh/#{hadoop_id_rsa_base}"
|
82
|
+
has_exec "chmod 600 #{home_dir}/.ssh/#{hadoop_id_rsa_base}"
|
83
|
+
has_exec "chmod 700 #{home_dir}/.ssh"
|
84
|
+
has_exec "rm -rf #{home_dir}/ssh"
|
85
|
+
|
86
|
+
# setup authorized keys
|
87
|
+
has_exec "touch #{home_dir}/.ssh/authorized_keys"
|
88
|
+
has_exec "chmod 644 #{home_dir}/.ssh/authorized_keys"
|
89
|
+
has_exec "chown -R #{user} #{home_dir}/.ssh"
|
90
|
+
has_line_in_file :file => "#{home_dir}/.ssh/authorized_keys", :line => File.read("#{hadoop_id_rsa}.pub")
|
91
|
+
end
|
92
|
+
|
93
|
+
def create_reference_hosts
|
94
|
+
clouds[:hadoop_master].nodes(:status => 'running').each_with_index do |n, i|
|
95
|
+
# has_host(:name => "master#{i}", :ip => n.public_ip) # todo
|
96
|
+
# my_line_in_file("/etc/hosts", "#{n.public_ip} master#{i}")
|
97
|
+
has_exec "ghost modify master#{i} \`dig +short #{n[:private_dns_name]}\`"
|
98
|
+
end
|
99
|
+
clouds[:hadoop_slave].nodes(:status => 'running').each_with_index do |n, i|
|
100
|
+
# has_host(:name => "slave#{i}", :ip => n.public_ip) # todo
|
101
|
+
# my_line_in_file("/etc/hosts", "#{n.public_ip} slave#{i}")
|
102
|
+
has_exec "ghost modify slave#{i} \`dig +short #{n[:private_dns_name]}\`"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def create_ssh_configs
|
107
|
+
ssh_config = ""
|
108
|
+
clouds[:hadoop_master].nodes(:status => 'running').each_with_index do |n,i|
|
109
|
+
has_exec "ssh -o 'StrictHostKeyChecking no' -i #{home_dir}/.ssh/#{hadoop_id_rsa_base} master#{i} echo || :", :user => user, # verify the host key
|
110
|
+
:only_if => "grep master#{i} /etc/hosts"
|
111
|
+
end
|
112
|
+
|
113
|
+
clouds[:hadoop_slave].nodes(:status => 'running').each_with_index do |n,i|
|
114
|
+
has_exec "ssh -o 'StrictHostKeyChecking no' -i #{home_dir}/.ssh/#{hadoop_id_rsa_base} slave#{i} echo || :", :user => user, # verify the host key
|
115
|
+
:only_if => "grep slave#{i} /etc/hosts"
|
116
|
+
end
|
117
|
+
|
118
|
+
ssh_config << <<EOF
|
119
|
+
Host *
|
120
|
+
IdentityFile #{home_dir}/.ssh/#{hadoop_id_rsa_base}
|
121
|
+
EOF
|
122
|
+
has_exec "ssh -o 'StrictHostKeyChecking no' -i #{home_dir}/.ssh/#{hadoop_id_rsa_base} localhost echo", :user => user # verify the host key
|
123
|
+
|
124
|
+
has_file("#{home_dir}/ssh_config", :content => ssh_config)
|
125
|
+
has_exec "mv #{home_dir}/ssh_config #{home_dir}/.ssh/config"
|
126
|
+
has_exec "chmod 600 #{home_dir}/.ssh/config"
|
127
|
+
has_exec "chown #{user}:#{group} #{home_dir}/.ssh/config"
|
128
|
+
end
|
129
|
+
|
130
|
+
def build
|
131
|
+
has_directory "/usr/local/src"
|
132
|
+
has_exec "wget http://www.gossipcheck.com/mirrors/apache/hadoop/core/hadoop-0.20.0/hadoop-0.20.0.tar.gz -O /usr/local/src/hadoop-0.20.0.tar.gz",
|
133
|
+
:not_if => "test -e /usr/local/src/hadoop-0.20.0.tar.gz"
|
134
|
+
has_exec "cd /usr/local/src && tar -xzf hadoop-0.20.0.tar.gz",
|
135
|
+
:not_if => "test -e #{hadoop_install_dir}"
|
136
|
+
has_exec "mv /usr/local/src/hadoop-0.20.0 /usr/local/src/hadoop",
|
137
|
+
:not_if => "test -e #{hadoop_install_dir}"
|
138
|
+
has_exec "chown -R #{user}:#{group} /usr/local/src/hadoop",
|
139
|
+
:not_if => "test -e #{hadoop_install_dir}"
|
140
|
+
has_exec "mv /usr/local/src/hadoop #{hadoop_install_dir}",
|
141
|
+
:not_if => "test -e #{hadoop_install_dir}"
|
142
|
+
# apply https://issues.apache.org/jira/secure/attachment/12407207/HADOOP-4675-v7.patch for ganglia 3.1 support
|
143
|
+
end
|
144
|
+
|
145
|
+
def hadoop_install_dir
|
146
|
+
"/usr/local/hadoop"
|
147
|
+
end
|
148
|
+
|
149
|
+
def configure
|
150
|
+
has_file(:name => hadoop_install_dir/"conf/hadoop-env.sh") do
|
151
|
+
mode 0644
|
152
|
+
template "hadoop-env.sh"
|
153
|
+
end
|
154
|
+
|
155
|
+
has_variable "current_master", :value => "master0" # todo, could eventually be made more dynamic here
|
156
|
+
|
157
|
+
# puts "we have this many nodes in our pool: #{number_of_running_nodes_in_pool}"
|
158
|
+
# has_variable "number_of_nodes", :value => lambda { %Q{ %x[/usr/bin/cloud-list --short].split("\\n").size || 1 }}
|
159
|
+
has_variable "number_of_nodes", :value => 2 # todo
|
160
|
+
|
161
|
+
has_directory hadoop_data_dir, :owner => user, :mode => "755"
|
162
|
+
has_exec "chgrp -R #{group} #{hadoop_data_dir}"
|
163
|
+
|
164
|
+
%w{logs name data mapred temp}.each do |folder|
|
165
|
+
has_directory hadoop_data_dir/folder, :owner => user, :mode => "755"
|
166
|
+
end
|
167
|
+
has_directory hadoop_data_dir/:temp/:dfs/:data, :owner => user, :mode => "755"
|
168
|
+
|
169
|
+
%w{local system temp}.each do |folder|
|
170
|
+
has_directory hadoop_data_dir/:temp/:mapred/folder, :owner => user, :mode => "755"
|
171
|
+
end
|
172
|
+
|
173
|
+
has_variable "hadoop_data_dir", :value => hadoop_data_dir
|
174
|
+
has_variable "hadoop_mapred_dir", :value => hadoop_data_dir/:mapred
|
175
|
+
|
176
|
+
has_variable("hadoop_this_nodes_ip", :value => lambda{ %Q{%x[curl http://169.254.169.254/latest/meta-data/local-ipv4]}})
|
177
|
+
|
178
|
+
%w{core hdfs mapred}.each do |config|
|
179
|
+
has_file(:name => hadoop_install_dir/"conf/#{config}-site.xml") do
|
180
|
+
mode 0644
|
181
|
+
template "#{config}-site.xml.erb"
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
has_file(:name => hadoop_install_dir/"conf/log4j.properties") do
|
186
|
+
mode 0644
|
187
|
+
template "log4j.properties.erb"
|
188
|
+
end
|
189
|
+
|
190
|
+
end
|
191
|
+
|
192
|
+
def number_of_running_nodes_in_pool
|
193
|
+
# clouds.keys.inject(0) { |sum,cloud_name| sum = sum + clouds[cloud_name].nodes(:status => 'running').size; sum }
|
194
|
+
end
|
195
|
+
|
196
|
+
def configure_master
|
197
|
+
# create_master_and_slaves_files
|
198
|
+
end
|
199
|
+
|
200
|
+
def format_hdfs
|
201
|
+
has_directory hadoop_data_dir, :mode => "770"
|
202
|
+
has_exec "chown -R #{user}:#{group} #{hadoop_data_dir}"
|
203
|
+
|
204
|
+
has_exec "#{hadoop_install_dir}/bin/hadoop namenode -format",
|
205
|
+
# :not_if => "test -e #{hadoop_data_dir}/hadoop-hadoop/dfs",
|
206
|
+
:not_if => "test -e #{hadoop_data_dir}/dfs/name", # this line depends on if you have user-based data directories in core-site.xml
|
207
|
+
:user => user
|
208
|
+
end
|
209
|
+
|
210
|
+
# stuff for examples
|
211
|
+
|
212
|
+
def prep_example_job
|
213
|
+
download_sample_data
|
214
|
+
end
|
215
|
+
|
216
|
+
def run_example_job
|
217
|
+
start_hadoop
|
218
|
+
copy_sample_data_to_hdfs
|
219
|
+
start_the_job
|
220
|
+
end
|
221
|
+
|
222
|
+
def start_hadoop
|
223
|
+
has_exec hadoop_install_dir/"bin/start-all.sh",
|
224
|
+
:user => user
|
225
|
+
end
|
226
|
+
|
227
|
+
def download_sample_data
|
228
|
+
has_directory "/tmp/gutenberg", :mode => "770", :owner => user
|
229
|
+
# todo, create has_wget
|
230
|
+
has_exec "wget http://www.gutenberg.org/files/20417/20417.txt -O /tmp/gutenberg/outline-of-science.txt",
|
231
|
+
:not_if => "test -e /tmp/gutenberg/outline-of-science.txt"
|
232
|
+
has_exec "wget http://www.gutenberg.org/dirs/etext04/7ldvc10.txt -O /tmp/gutenberg/7ldvc10.txt",
|
233
|
+
:not_if => "test -e /tmp/gutenberg/7ldvc10.txt"
|
234
|
+
has_exec "wget http://www.gutenberg.org/files/4300/4300.txt -O /tmp/gutenberg/ulysses.txt",
|
235
|
+
:not_if => "test -e /tmp/gutenberg/ulysses.txt"
|
236
|
+
has_exec "chown -R #{user}:#{group} /tmp/gutenberg"
|
237
|
+
end
|
238
|
+
|
239
|
+
def copy_sample_data_to_hdfs
|
240
|
+
has_exec "#{hadoop_install_dir}/bin/hadoop dfs -rmr gutenberg", :user => user,
|
241
|
+
:only_if => "sudo -H -u #{user} #{hadoop_install_dir}/bin/hadoop dfs -ls gutenberg"
|
242
|
+
has_exec "#{hadoop_install_dir}/bin/hadoop dfs -rmr gutenberg-output", :user => user,
|
243
|
+
:only_if => "sudo -H -u #{user} #{hadoop_install_dir}/bin/hadoop dfs -ls gutenberg-output"
|
244
|
+
has_exec "#{hadoop_install_dir}/bin/hadoop dfs -copyFromLocal /tmp/gutenberg gutenberg",
|
245
|
+
:not_if => "sudo -H -u #{user} #{hadoop_install_dir}/bin/hadoop dfs -ls gutenberg | grep ulysses",
|
246
|
+
:user => user
|
247
|
+
end
|
248
|
+
|
249
|
+
def start_the_job
|
250
|
+
has_exec "#{hadoop_install_dir}/bin/hadoop jar #{hadoop_install_dir}/hadoop-0.20.0-examples.jar wordcount gutenberg gutenberg-output",
|
251
|
+
:user => user
|
252
|
+
end
|
253
|
+
|
254
|
+
def create_master_and_slaves_files
|
255
|
+
masters_file = ""
|
256
|
+
slaves_file = ""
|
257
|
+
|
258
|
+
clouds[:hadoop_master].nodes(:status => 'running').each_with_index do |n,i|
|
259
|
+
masters_file << "master#{i}\n"
|
260
|
+
# slaves_file << "master#{i}\n" # our masters are also slaves
|
261
|
+
end
|
262
|
+
|
263
|
+
clouds[:hadoop_slave].nodes(:status => 'running').each_with_index do |n, i|
|
264
|
+
slaves_file << "slave#{i}\n"
|
265
|
+
end
|
266
|
+
|
267
|
+
has_file(hadoop_install_dir/:conf/:masters, :content => masters_file)
|
268
|
+
has_file(hadoop_install_dir/:conf/:slaves, :content => slaves_file)
|
269
|
+
end
|
270
|
+
|
271
|
+
def create_aliases
|
272
|
+
has_bash_alias :name => "cd-hadoop", :value => "pushd /usr/local/hadoop"
|
273
|
+
end
|
274
|
+
|
275
|
+
private
|
276
|
+
def cloud_keys_dir
|
277
|
+
File.dirname(pool_specfile)/:keys
|
278
|
+
end
|
279
|
+
|
280
|
+
def hadoop_id_rsa
|
281
|
+
"#{cloud_keys_dir}/#{hadoop_id_rsa_base}"
|
282
|
+
end
|
283
|
+
|
284
|
+
def hadoop_id_rsa_base
|
285
|
+
"hadoop_id_rsa"
|
286
|
+
end
|
287
|
+
|
288
|
+
def hadoop_data_dir
|
289
|
+
"/mnt/hadoop-data"
|
290
|
+
end
|
291
|
+
|
292
|
+
def home_dir
|
293
|
+
"/root"
|
294
|
+
# or
|
295
|
+
# "/home/hadoop"
|
296
|
+
end
|
297
|
+
|
298
|
+
def user
|
299
|
+
"root"
|
300
|
+
# or
|
301
|
+
# hadoop
|
302
|
+
end
|
303
|
+
|
304
|
+
def group
|
305
|
+
"root"
|
306
|
+
# or
|
307
|
+
# hadoop
|
308
|
+
end
|
309
|
+
|
310
|
+
def my_line_in_file(file, line)
|
311
|
+
has_exec "line_in_#{file}_#{line.safe_quote}" do
|
312
|
+
command "grep -q \'#{line.safe_quote}\' #{file} || echo \'#{line.safe_quote}\' >> #{file}"
|
313
|
+
not_if "grep -q \'#{line.safe_quote}\' #{file}"
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|