auser-poolparty-extensions 0.0.7 → 0.1.1

data/lib/extensions/hadoop/templates/init.d/hadoop-secondarynamenode

@@ -0,0 +1,124 @@
+#! /bin/sh
+#
+# skeleton  example file to build /etc/init.d/ scripts.
+#    This file should be used to construct scripts for /etc/init.d.
+#
+#    Written by Miquel van Smoorenburg <miquels@cistron.nl>.
+#    Modified for Debian
+#    by Ian Murdock <imurdock@gnu.ai.mit.edu>.
+#               Further changes by Javier Fernandez-Sanguino <jfs@debian.org>
+#
+# Version:  @(#)skeleton  1.9  26-Feb-2001  miquels@cistron.nl
+#
+### BEGIN INIT INFO
+# Provides:          hadoop-secondarynamenode
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:      $named
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Hadoop secondarynamenode daemon
+### END INIT INFO
+
+set -e
+
+# Include hadoop defaults if available
+if [ -f /usr/local/hadoop/conf/hadoop-env.sh ] ; then
+  . /usr/local/hadoop/conf/hadoop-env.sh
+fi
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
+NAME=hadoop-secondarynamenode
+DESC="Hadoop secondarynamenode daemon"
+
+test -x $DAEMON_SCRIPT || exit 0
+
+LOGDIR=$HADOOP_LOG_DIR
+PIDFILE=/var/run/hadoop/secondarynamenode.pid
+DODTIME=3                   # Time to wait for the server to die, in seconds
+                            # If this value is set too low you might not
+                            # let some servers to die gracefully and
+                            # 'restart' will not work
+
+get_running_pid() {
+    pid=$(ps axw -eo pid,command | tr 'A-Z' 'a-z' | grep org.apache.hadoop | grep secondarynamenode | grep java | awk '{print $1}')
+}
+
+running() {
+    get_running_pid
+    [ -z "$pid" ] && return 1
+    return 0
+}
+
+start() {
+    su -s /bin/sh root -c "$HADOOP_HOME/bin/hadoop-daemon.sh start secondarynamenode"
+}
+stop() {
+    su -s /bin/sh root -c "$HADOOP_HOME/bin/hadoop-daemon.sh stop secondarynamenode"
+}
+
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        if running ; then
+            echo "$NAME."
+            echo "Already running."
+        else
+            start
+            if running ; then
+                echo "$NAME."
+            else
+                echo "ERROR."
+            fi
+        fi
+  ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        stop
+        if ! running ; then
+            echo 'ERROR'
+        else
+            echo "$NAME."
+        fi
+  ;;
+  force-stop)
+  echo -n "Forcefully stopping $DESC: "
+        get_running_pid
+        kill -9 $pid
+        if ! running ; then
+            echo "$NAME."
+        else
+            echo " ERROR."
+        fi
+  ;;
+  force-reload)
+  # check wether $DAEMON is running. If so, restart
+        running && $0 restart
+  ;;
+  restart)
+        echo -n "Restarting $DESC: "
+        stop
+        [ -n "$DODTIME" ] && sleep $DODTIME
+        $0 start
+  ;;
+  status)
+    echo -n "$NAME is "
+    if running ;  then
+        echo "running"
+    else
+        echo "not running."
+        exit 1
+    fi
+    ;;
+  *)
+  N=/etc/init.d/$NAME
+  # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $N {start|stop|restart|force-reload|status|force-stop}" >&2
+  exit 1
+  ;;
+esac
+
+exit 0

data/lib/extensions/hadoop/templates/init.d/hadoop-tasktracker

@@ -0,0 +1,124 @@
+#! /bin/sh
+#
+# skeleton  example file to build /etc/init.d/ scripts.
+#    This file should be used to construct scripts for /etc/init.d.
+#
+#    Written by Miquel van Smoorenburg <miquels@cistron.nl>.
+#    Modified for Debian
+#    by Ian Murdock <imurdock@gnu.ai.mit.edu>.
+#               Further changes by Javier Fernandez-Sanguino <jfs@debian.org>
+#
+# Version:  @(#)skeleton  1.9  26-Feb-2001  miquels@cistron.nl
+#
+### BEGIN INIT INFO
+# Provides:          hadoop-tasktracker
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:      $named
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Hadoop tasktracker daemon
+### END INIT INFO
+
+set -e
+
+# Include hadoop defaults if available
+if [ -f /usr/local/hadoop/conf/hadoop-env.sh ] ; then
+  . /usr/local/hadoop/conf/hadoop-env.sh
+fi
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
+NAME=hadoop-tasktracker
+DESC="Hadoop tasktracker daemon"
+
+test -x $DAEMON_SCRIPT || exit 0
+
+LOGDIR=$HADOOP_LOG_DIR
+PIDFILE=/var/run/hadoop/tasktracker.pid
+DODTIME=3                   # Time to wait for the server to die, in seconds
+                            # If this value is set too low you might not
+                            # let some servers to die gracefully and
+                            # 'restart' will not work
+
+get_running_pid() {
+    pid=$(ps axw -eo pid,command | tr 'A-Z' 'a-z' | grep org.apache.hadoop | grep tasktracker | grep java | awk '{print $1}')
+}
+
+running() {
+    get_running_pid
+    [ -z "$pid" ] && return 1
+    return 0
+}
+
+start() {
+    su -s /bin/sh root -c "$HADOOP_HOME/bin/hadoop-daemon.sh start tasktracker"
+}
+stop() {
+    su -s /bin/sh root -c "$HADOOP_HOME/bin/hadoop-daemon.sh stop tasktracker"
+}
+
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        if running ; then
+            echo "$NAME."
+            echo "Already running."
+        else
+            start
+            if running ; then
+                echo "$NAME."
+            else
+                echo "ERROR."
+            fi
+        fi
+  ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        stop
+        if ! running ; then
+            echo 'ERROR'
+        else
+            echo "$NAME."
+        fi
+  ;;
+  force-stop)
+  echo -n "Forcefully stopping $DESC: "
+        get_running_pid
+        kill -9 $pid
+        if ! running ; then
+            echo "$NAME."
+        else
+            echo " ERROR."
+        fi
+  ;;
+  force-reload)
+  # check wether $DAEMON is running. If so, restart
+        running && $0 restart
+  ;;
+  restart)
+        echo -n "Restarting $DESC: "
+        stop
+        [ -n "$DODTIME" ] && sleep $DODTIME
+        $0 start
+  ;;
+  status)
+    echo -n "$NAME is "
+    if running ;  then
+        echo "running"
+    else
+        echo "not running."
+        exit 1
+    fi
+    ;;
+  *)
+  N=/etc/init.d/$NAME
+  # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+  echo "Usage: $N {start|stop|restart|force-reload|status|force-stop}" >&2
+  exit 1
+  ;;
+esac
+
+exit 0

data/lib/extensions/hadoop/templates/jvm.conf

@@ -0,0 +1,12 @@
+ # /etc/jvm
+ #
+ # This file defines the default system JVM search order. Each
+ # JVM should list their JAVA_HOME compatible directory in this file.
+ # The default system JVM is the first one available from top to
+ # bottom.
+ 
+ /usr/lib/jvm/java-6-sun
+ /usr/lib/jvm/java-gcj
+ /usr/lib/jvm/ia32-java-1.5.0-sun
+ /usr/lib/jvm/java-1.5.0-sun
+ /usr

data/lib/extensions/hadoop/templates/log4j.properties.erb

@@ -0,0 +1,94 @@
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# FSNamesystem Audit logging
+# All audit events are logged at INFO level
+#
+log4j.logger.org.apache.hadoop.fs.FSNamesystem.audit=WARN
+
+# Custom Logging levels
+
+log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter

data/lib/extensions/hadoop/templates/mapred-site.xml.erb

@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+  <name>mapred.job.tracker</name>
+  <value><%= @node[:poolparty][:current_master] %>:54311</value>
+  <description>The host and port that the MapReduce job tracker runs
+  at.  If "local", then jobs are run in-process as a single map
+  and reduce task.
+  </description>
+</property>
+
+
+<property>
+  <name>mapred.task.tracker.report.address</name>
+  <value><%= @node[:poolparty][:hadoop_this_nodes_ip] %>:0</value>
+  <description>The interface and port that task tracker server listens on.
+  Since it is only connected to by the tasks, it uses the local interface.
+  EXPERT ONLY. Should only be changed if your host does not have the loopback
+  interface.</description>
+</property>
+
+<property>
+  <name>mapred.local.dir</name>
+  <value><%= @node[:poolparty][:hadoop_data_dir] %>/mapred/local</value>
+</property>
+
+<property>
+  <name>mapred.system.dir</name>
+  <value><%= @node[:poolparty][:hadoop_data_dir] %>/mapred/system</value>
+</property>
+
+<property>
+  <name>mapred.tasktracker.dns.interface</name>
+  <value>eth0</value>
+</property>
+
+<property>
+  <name>mapred.job.tracker.persist.jobstatus.active</name>
+  <value>true</value>
+</property>
+
+
+
+
+
+<property>
+  <name>mapred.child.java.opts</name>
+  <value>-Xmx200m -Djava.net.preferIPv4Stack=true</value>
+  <description>Java opts for the task tracker child processes.
+  The following symbol, if present, will be interpolated: @taskid@ is replaced
+  by current TaskID. Any other occurrences of '@' will go unchanged.
+  For example, to enable verbose gc logging to a file named for the taskid in
+  /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+        -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+
+  TaskManager.VMFlags=-Djava.net.preferIPv6Addresses=true -Djava.net.preferIPv4Stack=false
+
+  The configuration variable mapred.child.ulimit can be used to control the
+  maximum virtual memory of the child processes.
+  </description>
+</property>
+
+
+
+</configuration>

data/lib/extensions/hive/hive.rb

@@ -0,0 +1,138 @@
+=begin rdoc
+In
+=end
+
+module PoolParty
+  module Plugin
+    class Hive < Plugin
+      def before_load(o={}, &block)
+        do_once do
+          # install_from_bin
+          install_from_src
+          set_environment_variables
+          create_hdfs_directories
+        end
+      end
+
+      def loaded(o={}, &block)
+        do_once do
+          has_file :name => "#{hive_home}/conf/hive-default.xml", :template => "hive-default.xml.erb"
+        end
+      end
+
+      def install_from_bin
+        has_exec "wget #{hive_dist} -O /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz",
+          :not_if => "test -e /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz"
+        has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz", 
+          :not_if => "test -e #{hive_home}"
+        has_exec "mv /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev #{hive_home}", 
+          :not_if => "test -e #{hive_home}"
+      end
+
+      # doesn't really work
+      def install_from_src
+        install_dependent_packages
+        download_and_build_src
+      end
+
+      def install_dependent_packages
+        has_package :name => "subversion"
+        has_package :name => "ant"
+      end
+
+      def download_and_build_src
+        has_exec "svn co #{hive_repo} #{src_dir} -r#{hive_revision}",
+          :not_if => "test -e #{src_dir}/build.xml"
+        has_exec "cd #{src_dir} && wget --no-check-certificate https://issues.apache.org/jira/secure/attachment/12409779/hive-487.3.patch",
+          :not_if => "test -e #{src_dir}/hive-487.3.patch"
+        has_exec "cd #{src_dir} && patch -p0 < hive-487.3.patch && mv hive-487.3.patch hive-487.3.patch.applied", 
+          :not_if => "test -e #{src_dir}/hive-487.3.patch.applied"
+        has_exec "cd #{src_dir} && ant -Dhadoop.version=\\\"#{hadoop_version}\\\" package",
+          :not_if => "test -e #{hive_home}/README.txt"
+        has_exec "mv #{src_dir}/build/dist #{hive_home}",
+          :not_if => "test -e #{hive_home}"
+      end
+
+      # todo, pull from parent
+      def set_environment_variables
+        has_file :name => "/root/.hadoop-etc-env.sh", :content => <<-EOF
+export HADOOP_HOME=#{hadoop_home}
+export HADOOP=$HADOOP_HOME/bin/hadoop
+export HIVE_HOME=#{hive_home}
+export PATH=$HADOOP_HOME/bin:$HIVE_HOME/bin:$PATH
+        EOF
+        has_line_in_file :file => "/root/.profile", :line => "source /root/.hadoop-etc-env.sh"
+      end
+
+      def create_hdfs_directories
+        has_exec "#{hadoop_home}/bin/hadoop fs -mkdir /tmp", 
+          :not_if => "#{hadoop_home}/bin/hadoop fs -ls /tmp",
+          :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
+
+        has_exec "#{hadoop_home}/bin/hadoop fs -mkdir /user/hive/warehouse", 
+          :not_if => "#{hadoop_home}/bin/hadoop fs -ls /user/hive/warehouse",
+          :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
+
+        has_exec "#{hadoop_home}/bin/hadoop fs -chmod g+w /tmp", 
+          :not_if => "#{hadoop_home}/bin/hadoop fs -ls /tmp", # todo, check perms
+          :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
+ 
+        has_exec "#{hadoop_home}/bin/hadoop fs -chmod g+w /user/hive/warehouse", 
+          :not_if => "#{hadoop_home}/bin/hadoop fs -ls /user/hive/warehouse",
+          :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
+      end
+
+      # you can also pass 'host:port' for host
+      def use_mysql(host, user, pass)
+        has_exec "wget http://mysql.mirrors.pair.com/Downloads/Connector-J/mysql-connector-java-5.0.8.tar.gz -O /usr/local/src/mysql-connector-java-5.0.8.tar.gz", 
+          :not_if => "test -e /usr/local/src/mysql-connector-java-5.0.8.tar.gz"
+        has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/mysql-connector-java-5.0.8.tar.gz", 
+          :not_if => "test -e /usr/local/src/mysql-connector-java-5.0.8"
+        has_exec "mv /usr/local/src/mysql-connector-java-5.0.8/mysql-connector-java-5.0.8-bin.jar #{hive_home}/lib", 
+          :not_if => "test -e #{hive_home}/lib/mysql-connector-java-5.0.8-bin.jar"
+        has_variable :name => "hive_mysql_enabled", :value => "true"
+        has_variable :name => "hive_mysql_host", :value => host
+        has_variable :name => "hive_mysql_user", :value => user
+        has_variable :name => "hive_mysql_pass", :value => pass
+      end
+
+      private
+
+      def hive_dist
+        "http://www.apache.org/dist/hadoop/hive/hive-0.3.0/hive-0.3.0-hadoop-0.19.0-dev.tar.gz"
+      end
+
+      def src_dir
+        "/usr/local/src/hive"
+      end
+
+      def hive_home
+        "/usr/local/hive"
+      end
+
+      def hive_repo
+        # "http://svn.apache.org/repos/asf/hadoop/hive/tags/release-0.3.0/"
+        "http://svn.apache.org/repos/asf/hadoop/hive/trunk"
+      end
+
+      def hive_revision
+        "781069"
+      end
+
+      ### TODO the values below should pull from parent e.g. the hadoop plugin
+      def hadoop_home
+        "/usr/local/hadoop"
+      end
+
+      def hadoop_data_dir
+        "/mnt/hadoop-data"
+      end
+
+      def hadoop_version
+        "0.20.0"
+      end
+
+    end
+  end
+end
+