jashmenn-poolparty-extensions 0.1.0 → 0.1.1

data/lib/extensions/hadoop/templates/core-site.xml.erb

@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ 
+<property>
+  <name>hadoop.tmp.dir</name>
+  <!--<value><%= @node[:poolparty][:hadoop_data_dir] %>/hadoop-${user.name}</value> -->
+  <value><%= @node[:poolparty][:hadoop_data_dir] %>/tmp</value>
+  <description>A base for other temporary directories.</description>
+</property>
+
+<property>
+  <name>fs.default.name</name>
+  <value>hdfs://<%= @node[:poolparty][:current_master] %>:54310</value>
+  <description>The name of the default file system.  A URI whose
+  scheme and authority determine the FileSystem implementation.  The
+  uri's scheme determines the config property (fs.SCHEME.impl) naming
+  the FileSystem implementation class.  The uri's authority is used to
+  determine the host, port, etc. for a filesystem.</description>
+</property>
+
+<property>
+  <name>webinterface.private.actions</name>
+  <value>true</value>
+</property>
+
+<!--
+   String jobTrackerStr =
+      conf.get("mapred.job.tracker", "localhost:8012");
+      -->
+
+<!-- take a look at below -->
+
+<!--
+<property>
+  <name>topology.node.switch.mapping.impl</name>
+  <value>org.apache.hadoop.net.ScriptBasedMapping</value>
+  <description> The default implementation of the DNSToSwitchMapping. It
+    invokes a script specified in topology.script.file.name to resolve
+    node names. If the value for topology.script.file.name is not set, the
+    default value of DEFAULT_RACK is returned for all node names.
+  </description>
+</property>
+
+<property>
+  <name>topology.script.file.name</name>
+  <value></value>
+  <description> The script name that should be invoked to resolve DNS names to
+    NetworkTopology names. Example: the script would take host.foo.bar as an
+    argument, and return /rack1 as the output.
+  </description>
+</property>
+-->
+
+<property>
+  <name>hadoop.rpc.socket.factory.class.default</name>
+  <value>org.apache.hadoop.net.StandardSocketFactory</value>
+  <final>true</final>
+</property>
+
+</configuration>

data/lib/extensions/hadoop/templates/hadoop-env.sh

@@ -0,0 +1,3 @@
+# The java implementation to use.  Required.
+export JAVA_HOME=/usr/lib/jvm/java-6-sun
+export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true

data/lib/extensions/hadoop/templates/hadoop-site.xml.erb

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ 
+<!-- Put site-specific property overrides in this file. -->
+ 
+<configuration>
+ 
+
+</configuration>

data/lib/extensions/hadoop/templates/hdfs-site.xml.erb

@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+  <name>dfs.replication</name>
+  <value><%= @node[:poolparty][:number_of_nodes] || 1 %></value>
+  <description>Default block replication.
+  The actual number of replications can be specified when the file is created.
+  The default is used if replication is not specified in create time.
+  </description>
+</property>
+
+<property>
+  <name>dfs.name.dir</name>
+  <value><%= @node[:poolparty][:hadoop_data_dir] %>/dfs/name</value>
+</property>
+
+<property>
+  <name>dfs.data.dir</name>
+  <value><%= @node[:poolparty][:hadoop_data_dir] %>/dfs/data</value>
+</property>
+
+<property>
+  <name>dfs.hosts</name>
+  <value>/usr/local/hadoop/conf/slaves</value>
+  <description>
+    When we have a number of users connecting to the hadoop cloud they may
+    accidentially run "start-all.sh" and then their local machine will try to
+    connect to the cluster. We limit the machines that can be a part of the
+    hdfs to be only those listed in the slaves file. See 
+    http://www.cloudera.com/blog/2008/12/03/securing-a-hadoop-cluster-through-a-gateway/
+  </description>
+</property>
+
+
+<!--
+<property>
+<name>dfs.permissions</name>
+<value>false</value>
+</property>
+-->
+
+</configuration>

data/lib/extensions/hadoop/templates/init.d/hadoop-datanode

@@ -0,0 +1,119 @@
+#! /bin/sh
+#
+# skeleton  example file to build /etc/init.d/ scripts.
+#    This file should be used to construct scripts for /etc/init.d.
+#
+#    Written by Miquel van Smoorenburg <miquels@cistron.nl>.
+#    Modified for Debian
+#    by Ian Murdock <imurdock@gnu.ai.mit.edu>.
+#               Further changes by Javier Fernandez-Sanguino <jfs@debian.org>
+#
+# Version:  @(#)skeleton  1.9  26-Feb-2001  miquels@cistron.nl
+#
+### BEGIN INIT INFO
+# Provides:          hadoop-datanode
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:      $named
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Hadoop datanode daemon
+### END INIT INFO
+
+set -e
+
+# Include hadoop defaults if available
+if [ -f /etc/default/hadoop ] ; then
+  . /etc/default/hadoop
+fi
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
+NAME=hadoop-datanode
+DESC="Hadoop datanode daemon"
+
+test -x $DAEMON_SCRIPT || exit 0
+
+LOGDIR=$HADOOP_LOG_DIR
+PIDFILE=/var/run/hadoop/datanode.pid
+DODTIME=3                   # Time to wait for the server to die, in seconds
+                            # If this value is set too low you might not
+                            # let some servers to die gracefully and
+                            # 'restart' will not work
+
+get_running_pid() {
+    pid=$(ps axw -eo pid,command | tr 'A-Z' 'a-z' | grep org.apache.hadoop | grep datanode | grep java | awk '{print $1}')
+}
+
+running() {
+    get_running_pid
+    [ -z "$pid" ] && return 1
+    return 0
+}
+
+start() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh start datanode"
+}
+stop() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh stop datanode"
+}
+
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        start
+        if running ; then
+            echo "$NAME."
+        else
+            echo "ERROR."
+        fi
+  ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        stop
+        if ! running ; then
+            echo 'ERROR'
+        else
+            echo "$NAME."
+        fi
+  ;;
+  force-stop)
+  echo -n "Forcefully stopping $DESC: "
+        get_running_pid
+        kill -9 $pid
+        if ! running ; then
+            echo "$NAME."
+        else
+            echo " ERROR."
+        fi
+  ;;
+  force-reload)
+  # check wether $DAEMON is running. If so, restart
+        running && $0 restart
+  ;;
+  restart)
+        echo -n "Restarting $DESC: "
+        stop
+        [ -n "$DODTIME" ] && sleep $DODTIME
+        $0 start
+  ;;
+  status)
+    echo -n "$NAME is "
+    if running ;  then
+        echo "running"
+    else
+        echo "not running."
+        exit 1
+    fi
+    ;;
+  *)
+  N=/etc/init.d/$NAME
+  # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $N {start|stop|restart|force-reload|status|force-stop}" >&2
+  exit 1
+  ;;
+esac
+
+exit 0

data/lib/extensions/hadoop/templates/init.d/hadoop-jobtracker

@@ -0,0 +1,119 @@
+#! /bin/sh
+#
+# skeleton  example file to build /etc/init.d/ scripts.
+#    This file should be used to construct scripts for /etc/init.d.
+#
+#    Written by Miquel van Smoorenburg <miquels@cistron.nl>.
+#    Modified for Debian
+#    by Ian Murdock <imurdock@gnu.ai.mit.edu>.
+#               Further changes by Javier Fernandez-Sanguino <jfs@debian.org>
+#
+# Version:  @(#)skeleton  1.9  26-Feb-2001  miquels@cistron.nl
+#
+### BEGIN INIT INFO
+# Provides:          hadoop-jobtracker
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:      $named
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Hadoop jobtracker daemon
+### END INIT INFO
+
+set -e
+
+# Include hadoop defaults if available
+if [ -f /etc/default/hadoop ] ; then
+  . /etc/default/hadoop
+fi
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
+NAME=hadoop-jobtracker
+DESC="Hadoop jobtracker daemon"
+
+test -x $DAEMON_SCRIPT || exit 0
+
+LOGDIR=$HADOOP_LOG_DIR
+PIDFILE=/var/run/hadoop/jobtracker.pid
+DODTIME=3                   # Time to wait for the server to die, in seconds
+                            # If this value is set too low you might not
+                            # let some servers to die gracefully and
+                            # 'restart' will not work
+
+get_running_pid() {
+    pid=$(ps axw -eo pid,command | tr 'A-Z' 'a-z' | grep org.apache.hadoop | grep jobtracker | grep java | awk '{print $1}')
+}
+
+running() {
+    get_running_pid
+    [ -z "$pid" ] && return 1
+    return 0
+}
+
+start() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh start jobtracker"
+}
+stop() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh stop jobtracker"
+}
+
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        start
+        if running ; then
+            echo "$NAME."
+        else
+            echo "ERROR."
+        fi
+  ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        stop
+        if ! running ; then
+            echo 'ERROR'
+        else
+            echo "$NAME."
+        fi
+  ;;
+  force-stop)
+  echo -n "Forcefully stopping $DESC: "
+        get_running_pid
+        kill -9 $pid
+        if ! running ; then
+            echo "$NAME."
+        else
+            echo " ERROR."
+        fi
+  ;;
+  force-reload)
+  # check wether $DAEMON is running. If so, restart
+        running && $0 restart
+  ;;
+  restart)
+        echo -n "Restarting $DESC: "
+        stop
+        [ -n "$DODTIME" ] && sleep $DODTIME
+        $0 start
+  ;;
+  status)
+    echo -n "$NAME is "
+    if running ;  then
+        echo "running"
+    else
+        echo "not running."
+        exit 1
+    fi
+    ;;
+  *)
+  N=/etc/init.d/$NAME
+  # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $N {start|stop|restart|force-reload|status|force-stop}" >&2
+  exit 1
+  ;;
+esac
+
+exit 0

data/lib/extensions/hadoop/templates/init.d/hadoop-namenode

@@ -0,0 +1,119 @@
+#! /bin/sh
+#
+# skeleton  example file to build /etc/init.d/ scripts.
+#    This file should be used to construct scripts for /etc/init.d.
+#
+#    Written by Miquel van Smoorenburg <miquels@cistron.nl>.
+#    Modified for Debian
+#    by Ian Murdock <imurdock@gnu.ai.mit.edu>.
+#               Further changes by Javier Fernandez-Sanguino <jfs@debian.org>
+#
+# Version:  @(#)skeleton  1.9  26-Feb-2001  miquels@cistron.nl
+#
+### BEGIN INIT INFO
+# Provides:          hadoop-namenode
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:      $named
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Hadoop namenode daemon
+### END INIT INFO
+
+set -e
+
+# Include hadoop defaults if available
+if [ -f /etc/default/hadoop ] ; then
+  . /etc/default/hadoop
+fi
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
+NAME=hadoop-namenode
+DESC="Hadoop namenode daemon"
+
+test -x $DAEMON_SCRIPT || exit 0
+
+LOGDIR=$HADOOP_LOG_DIR
+PIDFILE=/var/run/hadoop/namenode.pid
+DODTIME=3                   # Time to wait for the server to die, in seconds
+                            # If this value is set too low you might not
+                            # let some servers to die gracefully and
+                            # 'restart' will not work
+
+get_running_pid() {
+    pid=$(ps axw -eo pid,command | tr 'A-Z' 'a-z' | grep org.apache.hadoop | grep namenode | grep java | awk '{print $1}')
+}
+
+running() {
+    get_running_pid
+    [ -z "$pid" ] && return 1
+    return 0
+}
+
+start() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh start namenode"
+}
+stop() {
+    su -s /bin/sh hadoop -c "$HADOOP_HOME/bin/hadoop-daemon.sh stop namenode"
+}
+
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        start
+        if running ; then
+            echo "$NAME."
+        else
+            echo "ERROR."
+        fi
+  ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        stop
+        if ! running ; then
+            echo 'ERROR'
+        else
+            echo "$NAME."
+        fi
+  ;;
+  force-stop)
+  echo -n "Forcefully stopping $DESC: "
+        get_running_pid
+        kill -9 $pid
+        if ! running ; then
+            echo "$NAME."
+        else
+            echo " ERROR."
+        fi
+  ;;
+  force-reload)
+  # check wether $DAEMON is running. If so, restart
+        running && $0 restart
+  ;;
+  restart)
+        echo -n "Restarting $DESC: "
+        stop
+        [ -n "$DODTIME" ] && sleep $DODTIME
+        $0 start
+  ;;
+  status)
+    echo -n "$NAME is "
+    if running ;  then
+        echo "running"
+    else
+        echo "not running."
+        exit 1
+    fi
+    ;;
+  *)
+  N=/etc/init.d/$NAME
+  # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $N {start|stop|restart|force-reload|status|force-stop}" >&2
+  exit 1
+  ;;
+esac
+
+exit 0